From 31bb0e6612257384c11fcaa4ac6e190c2bcd0d82 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 10 May 2024 13:50:08 +0530 Subject: [PATCH 001/513] test pmc installation --- azure-pipelines.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index db5200337..fb3f2ccc2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,17 @@ pr: - main jobs: + - job: PMC Test + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + steps: + - checkout: none + - task: PipAuthenticate@1 + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' - job: Build timeoutInMinutes: 360 strategy: From 0a17f372a09dee1fcbafdf62e82a1353f357d77f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 10 May 2024 13:50:08 +0530 Subject: [PATCH 002/513] test pmc installation --- azure-pipelines.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index db5200337..0568ff03b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,17 @@ pr: - main jobs: + - job: PMC_Test + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + steps: + - checkout: none + - task: PipAuthenticate@1 + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' - job: Build timeoutInMinutes: 360 strategy: From bb154f13b2414a173c7e7941ba7d0ea8f3db6129 Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Fri, 10 May 2024 14:48:05 +0530 Subject: [PATCH 003/513] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 643a0d137..0c8935364 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,7 @@ jobs: - task: PipAuthenticate@1 inputs: artifactFeeds: 'AzCopy-NextGen/AzCopyFeed' - displayName: 'Connect to PMC artifact' + displayName: 'Connect to PMC artifact' - script: | pip install pmc-cli From 75e8944e685ffae651883b1cc8492810c4f12ab2 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Fri, 10 May 2024 22:07:34 +0530 Subject: [PATCH 004/513] Update azure-pipelines.yml for Azure Pipelines Change feed name --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 431fa8dc8..125d758f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,7 +23,7 @@ jobs: - checkout: none - task: PipAuthenticate@1 inputs: - artifactFeeds: 'BlobFuse/Blobfuse' + artifactFeeds: 'azstorage/DevExGlobalFeed' displayName: 'Connect to PMC artifact' - script: | From 62033d9397d335f587667befa03ca0515385ca3f Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Fri, 10 May 2024 22:09:41 +0530 Subject: [PATCH 005/513] Update azure-pipelines.yml for Azure Pipelines . --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 125d758f3..0954a8929 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,7 +23,7 @@ jobs: - checkout: none - task: PipAuthenticate@1 inputs: - artifactFeeds: 'azstorage/DevExGlobalFeed' + artifactFeeds: 'DevExGlobalFeed' displayName: 'Connect to PMC artifact' - script: | From 75e7587c5e1d5ddb313e472d1b14d852b4ab68d0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 13 May 2024 13:31:14 +0530 Subject: [PATCH 006/513] testing pmc installation --- release-pipelines.yml | 400 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 release-pipelines.yml diff --git a/release-pipelines.yml b/release-pipelines.yml new file mode 100644 index 000000000..767820f5a --- /dev/null +++ b/release-pipelines.yml @@ -0,0 +1,400 @@ +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' + +trigger: + branches: + include: + - dev + - main + +pr: + branches: + include: + - dev + - main + +jobs: + - job: PMC_Test + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + steps: + - checkout: none + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install -vvv pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + +# - job: Build +# timeoutInMinutes: 360 +# strategy: +# matrix: +# Ubuntu-20: +# imageName: 'ubuntu-20.04' +# type: 'linux' +# MacOS: +# imageName: 'macos-11' +# type: 'mac-os' +# Windows: +# imageName: 'windows-2019' +# type: 'windows' +# pool: +# vmImage: $(imageName) +# steps: +# - task: GoTool@0 +# env: +# GO111MODULE: 'on' +# inputs: +# version: $(AZCOPY_GOLANG_VERSION) + +# - script: | +# echo 'Running GO Vet' +# go vet +# displayName: 'Golang Vet - Linux' +# workingDirectory: $(System.DefaultWorkingDirectory) + +# - script: | +# GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64" +# displayName: 'Generate Linux AMD64' +# condition: eq(variables.type, 'linux') + +# - script: | +# GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" +# displayName: 'Generate Linux AMD64 SE Integration' +# condition: eq(variables.type, 'linux') + +# - script: | +# GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" +# displayName: 'Generate Linux ARM64' +# condition: eq(variables.type, 'linux') + +# - script: | +# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" +# displayName: 'Generate Windows AMD64' +# condition: eq(variables.type, 'windows') +# env: +# GOARCH: amd64 +# GOOS: windows +# CGO_ENABLED: 0 + +# - script: | +# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" +# displayName: 'Generate Windows i386' +# condition: eq(variables.type, 'windows') +# env: +# GOARCH: 386 +# GOOS: windows +# CGO_ENABLED: 0 + +# - script: | +# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" +# displayName: 'Generate Windows ARM' +# condition: eq(variables.type, 'windows') +# env: +# GOARCH: arm +# GOARM: 7 +# GOOS: windows +# CGO_ENABLED: 0 + +# - script: | +# cp NOTICE.txt $(Build.ArtifactStagingDirectory) +# displayName: 'Copy NOTICE.txt' +# condition: eq(variables.type, 'linux') + +# - script: | +# CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" +# displayName: 'Generate MacOS Build with AMD64' +# condition: eq(variables.type, 'mac-os') + +# # uncomment below when manually releasing for m1 +# # - script: | +# # curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" +# # displayName: 'Generate MacOS Build with ARM64' +# # condition: eq(variables.type, 'mac-os') + +# # cross compile to make sure nothing broke +# - script: | +# GOARCH=arm64 CGO_ENABLED=1 go build +# displayName: 'Test Cross-compiled MacOS Build with ARM64' +# condition: eq(variables.type, 'mac-os') + +# - task: PublishBuildArtifacts@1 +# displayName: 'Publish Artifacts' +# condition: succeededOrFailed() + +# - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) + + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' + + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs + + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' + + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + + # - job: Test_On_Ubuntu + variables: + isMutexSet: 'false' + # allow maximum build time, in case we have build congestion + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + name: 'Set_up_Python' + inputs: + versionSpec: '3.7' + - task: GoTool@0 + name: 'Set_up_Golang' + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - task: DownloadSecureFile@1 + name: ciGCSServiceAccountKey + displayName: 'Download GCS Service Account Key' + inputs: + secureFile: 'ci-gcs-dev.json' + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Install dependencies' + - script: | + pip install azure-storage-blob==12.12.0 + # set the variable to indicate that the mutex is being acquired + # note: we set it before acquiring the mutex to ensure we release the mutex. + # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # mutex but before we set this variable. + # setting this before will always work since it is valid to break an un-acquired mutex. + echo '##vso[task.setvariable variable=isMutexSet]true' + # acquire the mutex before running live tests to avoid conflicts + python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + name: 'Acquire_the_distributed_mutex' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'cmd' + coverage_name: 'cmd' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common' + coverage_name: 'common' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common/parallel' + coverage_name: 'parallel' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'ste' + coverage_name: 'ste' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'sddl' + coverage_name: 'sddl' + - script: | + GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + go build -tags "netgo" -o test-validator ./testSuite/ + mkdir test-temp + mkdir coverage + export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + keyctl session test python ./testSuite/scripts/run.py + + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o smoke_coverage.txt + echo 'Formatting coverage to json format' + $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + echo 'Formatting coverage to xml format' + $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + name: 'Run_smoke_tests' + env: + ACCOUNT_NAME: $(ACCOUNT_NAME) + ACCOUNT_KEY: $(ACCOUNT_KEY) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + FILESYSTEM_URL: $(FILESYSTEM_URL) + OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + SHARE_SAS_URL: $(SHARE_SAS_URL) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + condition: succeededOrFailed() + # Smoke Tests Publishing + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + - script: | + pip install azure-storage-blob==12.12.0 + python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + name: 'Release_the_distributed_mutex' + # this runs even if the job was canceled (only if the mutex was acquired by this job) + condition: and(always(), eq(variables['isMutexSet'], 'true')) \ No newline at end of file From 2044c3bda2e61678e2a7d39d3f3c6d54946c0ee6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 13 May 2024 17:29:30 +0530 Subject: [PATCH 007/513] Added Build artifacts stage --- release-pipelines.yml | 1891 ++++++++++++++++++++++++++++++++--------- 1 file changed, 1494 insertions(+), 397 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 44465762d..f9a34303b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1,400 +1,1497 @@ -variables: - AZCOPY_GOLANG_VERSION: '1.21.8' - AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' - -trigger: - branches: - include: - - dev - - main - -pr: - branches: - include: - - dev - - main - -jobs: - - job: PMC_Test - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' - steps: - - checkout: none - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'DevExGlobalFeed' - displayName: 'Connect to PMC artifact' +parameters: + - name: tag + displayName: 'New Release Tag' + type: string + default: 'azcopy-' + + - name: unit_test + displayName: 'Execute Unit Tests' + type: boolean + default: false + + - name: post_release + displayName: 'Post Release on Github' + type: boolean + default: false + + - name: publish_artifacts + displayName: 'Publish Artifacts to Linux Repos' + type: boolean + default: false + + - name: draft + displayName: 'Post as Draft Release' + type: boolean + default: false + + - name: prerelease + displayName: 'Post as PreRelease' + type: boolean + default: false + + - name: update_version + displayName: 'Update Version' + type: boolean + default: false + +# Do not trigger this pipeline automatically +trigger: none +pr: none + +stages: + - stage: BuildArtifacts + jobs: + - job: Build + timeoutInMinutes: 360 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + type: 'linux' + MacOS: + imageName: 'macos-11' + type: 'mac-os' + Windows: + imageName: 'windows-2019' + type: 'windows' + + pool: + vmImage: $(imageName) + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + condition: eq(variables.type, 'linux') + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + condition: eq(variables.type, 'linux') + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + condition: eq(variables.type, 'linux') + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + condition: eq(variables.type, 'windows') + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + condition: eq(variables.type, 'windows') + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + condition: eq(variables.type, 'windows') + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + cp NOTICE.txt $(Build.ArtifactStagingDirectory) + displayName: 'Copy NOTICE.txt' + condition: eq(variables.type, 'linux') + + - script: | + CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + condition: eq(variables.type, 'mac-os') + + # uncomment below when manually releasing for m1 +# - script: | +# curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" +# displayName: 'Generate MacOS Build with ARM64' +# condition: eq(variables.type, 'mac-os') + + # cross compile to make sure nothing broke + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build + displayName: 'Test Cross-compiled MacOS Build with ARM64' + condition: eq(variables.type, 'mac-os') + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - - script: | - pip install -vvv pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - -# - job: Build -# timeoutInMinutes: 360 -# strategy: -# matrix: -# Ubuntu-20: -# imageName: 'ubuntu-20.04' -# type: 'linux' -# MacOS: -# imageName: 'macos-11' -# type: 'mac-os' -# Windows: -# imageName: 'windows-2019' -# type: 'windows' -# pool: -# vmImage: $(imageName) -# steps: -# - task: GoTool@0 -# env: -# GO111MODULE: 'on' -# inputs: -# version: $(AZCOPY_GOLANG_VERSION) - -# - script: | -# echo 'Running GO Vet' -# go vet -# displayName: 'Golang Vet - Linux' -# workingDirectory: $(System.DefaultWorkingDirectory) - -# - script: | -# GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64" -# displayName: 'Generate Linux AMD64' -# condition: eq(variables.type, 'linux') - -# - script: | -# GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" -# displayName: 'Generate Linux AMD64 SE Integration' -# condition: eq(variables.type, 'linux') - -# - script: | -# GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" -# displayName: 'Generate Linux ARM64' -# condition: eq(variables.type, 'linux') - -# - script: | -# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" -# displayName: 'Generate Windows AMD64' -# condition: eq(variables.type, 'windows') -# env: -# GOARCH: amd64 -# GOOS: windows -# CGO_ENABLED: 0 - -# - script: | -# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" -# displayName: 'Generate Windows i386' -# condition: eq(variables.type, 'windows') -# env: -# GOARCH: 386 -# GOOS: windows -# CGO_ENABLED: 0 - -# - script: | -# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" -# displayName: 'Generate Windows ARM' -# condition: eq(variables.type, 'windows') -# env: -# GOARCH: arm -# GOARM: 7 -# GOOS: windows -# CGO_ENABLED: 0 - -# - script: | -# cp NOTICE.txt $(Build.ArtifactStagingDirectory) -# displayName: 'Copy NOTICE.txt' -# condition: eq(variables.type, 'linux') - -# - script: | -# CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" -# displayName: 'Generate MacOS Build with AMD64' -# condition: eq(variables.type, 'mac-os') - -# # uncomment below when manually releasing for m1 -# # - script: | -# # curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" -# # displayName: 'Generate MacOS Build with ARM64' -# # condition: eq(variables.type, 'mac-os') - -# # cross compile to make sure nothing broke -# - script: | -# GOARCH=arm64 CGO_ENABLED=1 go build -# displayName: 'Test Cross-compiled MacOS Build with ARM64' -# condition: eq(variables.type, 'mac-os') - -# - task: PublishBuildArtifacts@1 -# displayName: 'Publish Artifacts' -# condition: succeededOrFailed() +# BuildArtifacts end here + + # - stage: SignArtifacts + # dependsOn: BuildArtifacts + # condition: succeeded('BuildArtifacts') + # jobs: + # - job: SignBlobfuse + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + + # steps: + # - checkout: none + + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp blobfuse2-temp/*-fuse3*.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # # Send images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning blobfuse2' + # inputs: + # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/blobfuse2-temp' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_BLOBFUSE_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning blobfuse2 mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_BLOBFUSE_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm + # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/*.rpm + # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/blobfuse2-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/*.rpm + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2-signed' + # displayName: 'Publish Signed Artifacts' + # # SignArtifacts end here + + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: + # - job: Set_0 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo dpkg -i blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # Create container for blobfuse and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh Dockerfile x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + # ./buildcontainer.sh DockerfileMariner mariner-x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + # displayName: "Create and publish container" + # workingDirectory: $(work_dir)/docker + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-18: + # vmImage: 'Ubuntu-18.04' + # fuselib: 'libfuse-dev' + # fuse-version: 'fuse' + # tags: 'fuse2' + # container: 'test-cnt-ubn-18' + # AgentName: "blobfuse-ubuntu18" + # Ubuntu-20: + # vmImage: 'Ubuntu-20.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # if [ $(AgentName) == "blobfuse-ubuntu18" ]; then + # echo "Generating for Debian 9/10" + # f=`ls ./blobfuse2*$(vmImage)*.deb` + # cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") + # cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") + # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + # ls -l $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*Debian-*.deb + # else + # echo "Generating for Debian 11" + # f=`ls ./blobfuse2*$(vmImage)*.deb` + # cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") + # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + # ls -l $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*Debian-*.deb + # fi + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # sudo dpkg -i blobfuse2*.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_2_ARM + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22-ARM64: + # vmImage: 'Ubuntu-22.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*arm64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # sudo dpkg -i blobfuse2*.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # Create container for blobfuse and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh DockerfileARM arm64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) arm64 + # displayName: "Create and publish ARM container" + # workingDirectory: $(work_dir)/docker + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_2_AARCH + # timeoutInMinutes: 120 + # strategy: + # matrix: + # RHEL-9.0: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # fuselib: 'fuse fuse3-libs fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-9-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*aarch64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip blobfuse2*$(vmImage)*.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing + # sudo rpm -i blobfuse2*$(vmImage)*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # RHEL-7.5: + # agentName: "blobfuse-rhel7_5" + # vmImage: 'RHEL-7.5' + # fuselib: 'fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-75' + # RHEL-8.6: + # agentName: "blobfuse-rhel8_6" + # vmImage: 'RHEL-8.6' + # fuselib: 'fuse fuse3-libs fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-86' + # RHEL-9.0: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # fuselib: 'fuse fuse3-libs fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-9' + + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo touch /etc/yum.repos.d/centos.repo + # sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' + # condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) + # displayName: "Update OS mirrors" + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # if [ $(agentName) == "blobfuse-rhel7_5" ]; then + # f=`ls ./blobfuse2*$(vmImage)*.rpm` + # cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") + # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") + # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") + # cp ./blobfuse2*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) + # cp ./blobfuse2*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*RHEL-7.8*.rpm + # rm -rf ./blobfuse2*RHEL-8*.rpm + # fi + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip blobfuse2*$(vmImage)*.rpm + # sudo yum groupinstall "Development Tools" -y + # if [[ $(agentName) == "blobfuse-rhel7_5" || $(agentName) == "blobfuse-rhel7_8" ]]; then + # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y + # else + # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing + # fi + # sudo rpm -i blobfuse2*$(vmImage)*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # CentOS-7.9: + # agentName: "blobfuse-centos7" + # vmImage: 'CentOS-7.0' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-cent-7' + # CentOS-8.5: + # agentName: "blobfuse-centos8" + # vmImage: 'CentOS-8.0' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-cent-8' + + # pool: + # name: "blobfuse-centos-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + # sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + # condition: eq(variables['agentName'], 'blobfuse-centos8') + # displayName: "Update OS mirrors" + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo yum install gcc gcc-c++ make -y + # if [ $(agentName) == "blobfuse-centos8" ]; then + # sudo yum install fuse fuse3 fuse3-devel -y --nobest --allowerasing + # else + # sudo yum install fuse fuse3 fuse3-devel -y + # fi + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_6 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Oracle-8.1: + # agentName: "blobfuse-oracle81" + # vmImage: 'Oracle-8.1' + # fuselib: 'fuse3 fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: "test-cnt-ora-81" + + # pool: + # name: "blobfuse-oracle-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo yum install gcc gcc-c++ make -y + # sudo yum install fuse $(fuse-version) -y --nobest --allowerasing + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # SUSE-15: + # agentName: "blobfuse-suse15" + # vmImage: 'SUSE-15Gen2' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: "test-cnt-suse-15" -# - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) - - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - - # # - job: Test_On_Ubuntu - # variables: - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py + # pool: + # name: "blobfuse-suse-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo zypper update -y + # sudo zypper -n install git + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo zypper -n install make cmake gcc gcc-c++ fuse fuse3 + # wget https://rpmfind.net/linux/opensuse/distribution/leap/15.2/repo/oss/x86_64/fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm + # sudo zypper -n --no-gpg-checks install fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm + # sudo rm fuse3-devel*.rpm + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_8 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # fuselib: 'libfuse3-dev' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/mariner/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed/mariner + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo tdnf install gcc build-essential fuse3 fuse3-devel -y + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/blobfuse2-signed/mariner + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # TestArtifacts ends here + # - stage: ReleaseArtifacts + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') + # jobs: + # - job: ReleaseBlobfuse + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + + # steps: + # - checkout: none + + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.rpm + # displayName: 'List Artifacts' + + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - ${{ if eq(parameters.post_release, true) }}: + # # add release tags & push to github + # - task: GithubRelease@1 + # inputs: + # githubConnection: 'blobfuse-git-rel' + # repositoryName: 'Azure/azure-storage-fuse' + # action: 'edit' + + # target: '$(Build.SourceVersion)' + # tagSource: 'userSpecifiedTag' + + # title: ${{ parameters.tag }} + # tag: ${{ parameters.tag }} + + # assets: | + # $(Build.ArtifactStagingDirectory)/blobfuse2/* + # changeLogCompareToRelease: 'lastFullRelease' + # changeLogType: 'commitBased' + # isDraft: ${{ parameters.draft }} + # isPreRelease: ${{ parameters.prerelease }} + # assetUploadMode: replace + + # - ${{ if eq(parameters.publish_artifacts, true) }}: + # - stage: PublishArtifacts + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: PublishArtifacts + # timeoutInMinutes: 120 + # pool: + # vmImage: 'ubuntu-22.04' + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) \ No newline at end of file + # steps: + # - checkout: none + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'BlobFuse/Blobfuse' + # displayName: 'Connect to PMC artifact' + + # - script: | + # pip install pmc-cli + # displayName: 'Install pmc-cli' + + # - task: DownloadSecureFile@1 + # name: pmcCertificate + # displayName: 'Download pmc pem file' + # inputs: + # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + # - task: DownloadSecureFile@1 + # name: settings + # displayName: 'Download settings.toml file' + # inputs: + # secureFile: 'settings.toml' + + # - script: | + # pmc --version + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + # if [ $? -ne 0 ]; then + # exit 1 + # fi + # displayName: 'Test PMC installation' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Signed Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # cd mariner + # for f in ./blobfuse2*fuse3*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + # ls -lRt + # mv blobfuse2*.rpm .. + # cd .. + # rm -r mariner/ + # displayName: 'Rename Mariner binaries' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # displayName: 'List Artifacts' + + # - script: | + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload blobfuse2-signed + # displayName: 'Upload packages' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv + # cat ./packages.csv + # displayName: 'Fetch packages.csv' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` + # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + + # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` + # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" + + # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` + # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" + + # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` + # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" + + # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` + # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" + + # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` + # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" + + # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` + # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" + + # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` + # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" + + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Add uploaded packages to repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - script: | + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - stage: UpdateLatestVersion + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: UpdateVersion + # pool: + # vmImage: 'ubuntu-20.04' + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + + # steps: + # - checkout: none + + # - ${{ if eq(parameters.update_version, true) }}: + # - script: | + # sudo apt-get install python3 -y + # python3 --version + # displayName: 'Installing Python' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # # install blobfuse2 + # - script: | + # cd $(Build.ArtifactStagingDirectory)/blobfuse2 + # ls | grep -i ubuntu-20.04 + # sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y + # blobfuse2 version + # displayName: 'Installing blobfuse2' + + # - script: | + # wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py + # ls -l + # displayName: 'Getting Python script' + # workingDirectory: $(root_dir) + + # - script: | + # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`blobfuse2 version`" + # displayName: 'Updating version number' + # workingDirectory: $(root_dir) + From 296de4c9a4752d6dfcd403bd6c29c29f5a57b812 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 13 May 2024 17:29:30 +0530 Subject: [PATCH 008/513] Added Build artifacts stage --- release-pipelines.yml | 1891 ++++++++++++++++++++++++++++++++--------- 1 file changed, 1494 insertions(+), 397 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 44465762d..220293b5b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1,400 +1,1497 @@ -variables: - AZCOPY_GOLANG_VERSION: '1.21.8' - AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' - -trigger: - branches: - include: - - dev - - main - -pr: - branches: - include: - - dev - - main - -jobs: - - job: PMC_Test - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' - steps: - - checkout: none - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'DevExGlobalFeed' - displayName: 'Connect to PMC artifact' +parameters: + - name: tag + displayName: 'New Release Tag' + type: string + default: 'azcopy-' + + - name: unit_test + displayName: 'Execute Unit Tests' + type: boolean + default: false + + - name: post_release + displayName: 'Post Release on Github' + type: boolean + default: false + + - name: publish_artifacts + displayName: 'Publish Artifacts to Linux Repos' + type: boolean + default: false + + - name: draft + displayName: 'Post as Draft Release' + type: boolean + default: false + + - name: prerelease + displayName: 'Post as PreRelease' + type: boolean + default: false + + - name: update_version + displayName: 'Update Version' + type: boolean + default: false + +# Do not trigger this pipeline automatically +trigger: none +pr: none + +stages: + - stage: BuildArtifacts + jobs: + - job: Build + timeoutInMinutes: 360 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + type: 'linux' + MacOS: + imageName: 'macos-11' + type: 'mac-os' + Windows: + imageName: 'windows-2019' + type: 'windows' + + pool: + vmImage: $(imageName) + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + condition: eq(variables.type, 'linux') + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + condition: eq(variables.type, 'linux') + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + condition: eq(variables.type, 'linux') + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + condition: eq(variables.type, 'windows') + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + condition: eq(variables.type, 'windows') + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + condition: eq(variables.type, 'windows') + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + cp NOTICE.txt $(Build.ArtifactStagingDirectory) + displayName: 'Copy NOTICE.txt' + condition: eq(variables.type, 'linux') + + - script: | + CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + condition: eq(variables.type, 'mac-os') + + # uncomment below when manually releasing for m1 +# - script: | +# curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" +# displayName: 'Generate MacOS Build with ARM64' +# condition: eq(variables.type, 'mac-os') + + # cross compile to make sure nothing broke + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build + displayName: 'Test Cross-compiled MacOS Build with ARM64' + condition: eq(variables.type, 'mac-os') + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - - script: | - pip install -vvv pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - -# - job: Build -# timeoutInMinutes: 360 -# strategy: -# matrix: -# Ubuntu-20: -# imageName: 'ubuntu-20.04' -# type: 'linux' -# MacOS: -# imageName: 'macos-11' -# type: 'mac-os' -# Windows: -# imageName: 'windows-2019' -# type: 'windows' -# pool: -# vmImage: $(imageName) -# steps: -# - task: GoTool@0 -# env: -# GO111MODULE: 'on' -# inputs: -# version: $(AZCOPY_GOLANG_VERSION) - -# - script: | -# echo 'Running GO Vet' -# go vet -# displayName: 'Golang Vet - Linux' -# workingDirectory: $(System.DefaultWorkingDirectory) - -# - script: | -# GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64" -# displayName: 'Generate Linux AMD64' -# condition: eq(variables.type, 'linux') - -# - script: | -# GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" -# displayName: 'Generate Linux AMD64 SE Integration' -# condition: eq(variables.type, 'linux') - -# - script: | -# GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" -# displayName: 'Generate Linux ARM64' -# condition: eq(variables.type, 'linux') - -# - script: | -# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" -# displayName: 'Generate Windows AMD64' -# condition: eq(variables.type, 'windows') -# env: -# GOARCH: amd64 -# GOOS: windows -# CGO_ENABLED: 0 - -# - script: | -# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" -# displayName: 'Generate Windows i386' -# condition: eq(variables.type, 'windows') -# env: -# GOARCH: 386 -# GOOS: windows -# CGO_ENABLED: 0 - -# - script: | -# go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" -# displayName: 'Generate Windows ARM' -# condition: eq(variables.type, 'windows') -# env: -# GOARCH: arm -# GOARM: 7 -# GOOS: windows -# CGO_ENABLED: 0 - -# - script: | -# cp NOTICE.txt $(Build.ArtifactStagingDirectory) -# displayName: 'Copy NOTICE.txt' -# condition: eq(variables.type, 'linux') - -# - script: | -# CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" -# displayName: 'Generate MacOS Build with AMD64' -# condition: eq(variables.type, 'mac-os') - -# # uncomment below when manually releasing for m1 -# # - script: | -# # curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" -# # displayName: 'Generate MacOS Build with ARM64' -# # condition: eq(variables.type, 'mac-os') - -# # cross compile to make sure nothing broke -# - script: | -# GOARCH=arm64 CGO_ENABLED=1 go build -# displayName: 'Test Cross-compiled MacOS Build with ARM64' -# condition: eq(variables.type, 'mac-os') - -# - task: PublishBuildArtifacts@1 -# displayName: 'Publish Artifacts' -# condition: succeededOrFailed() +# BuildArtifacts end here + + # - stage: SignArtifacts + # dependsOn: BuildArtifacts + # condition: succeeded('BuildArtifacts') + # jobs: + # - job: SignBlobfuse + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + + # steps: + # - checkout: none + + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp blobfuse2-temp/*-fuse3*.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # # Send images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning blobfuse2' + # inputs: + # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/blobfuse2-temp' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_BLOBFUSE_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning blobfuse2 mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_BLOBFUSE_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm + # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/*.rpm + # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/blobfuse2-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/*.rpm + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2-signed' + # displayName: 'Publish Signed Artifacts' + # # SignArtifacts end here + + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: + # - job: Set_0 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo dpkg -i blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # Create container for blobfuse and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh Dockerfile x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + # ./buildcontainer.sh DockerfileMariner mariner-x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + # displayName: "Create and publish container" + # workingDirectory: $(work_dir)/docker + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-18: + # vmImage: 'Ubuntu-18.04' + # fuselib: 'libfuse-dev' + # fuse-version: 'fuse' + # tags: 'fuse2' + # container: 'test-cnt-ubn-18' + # AgentName: "blobfuse-ubuntu18" + # Ubuntu-20: + # vmImage: 'Ubuntu-20.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # if [ $(AgentName) == "blobfuse-ubuntu18" ]; then + # echo "Generating for Debian 9/10" + # f=`ls ./blobfuse2*$(vmImage)*.deb` + # cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") + # cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") + # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + # ls -l $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*Debian-*.deb + # else + # echo "Generating for Debian 11" + # f=`ls ./blobfuse2*$(vmImage)*.deb` + # cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") + # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + # ls -l $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*Debian-*.deb + # fi + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # sudo dpkg -i blobfuse2*.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_2_ARM + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22-ARM64: + # vmImage: 'Ubuntu-22.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*arm64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # sudo dpkg -i blobfuse2*.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # Create container for blobfuse and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh DockerfileARM arm64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) arm64 + # displayName: "Create and publish ARM container" + # workingDirectory: $(work_dir)/docker + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_2_AARCH + # timeoutInMinutes: 120 + # strategy: + # matrix: + # RHEL-9.0: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # fuselib: 'fuse fuse3-libs fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-9-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*aarch64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip blobfuse2*$(vmImage)*.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing + # sudo rpm -i blobfuse2*$(vmImage)*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # RHEL-7.5: + # agentName: "blobfuse-rhel7_5" + # vmImage: 'RHEL-7.5' + # fuselib: 'fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-75' + # RHEL-8.6: + # agentName: "blobfuse-rhel8_6" + # vmImage: 'RHEL-8.6' + # fuselib: 'fuse fuse3-libs fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-86' + # RHEL-9.0: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # fuselib: 'fuse fuse3-libs fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-rhel-9' + + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo touch /etc/yum.repos.d/centos.repo + # sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' + # condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) + # displayName: "Update OS mirrors" + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # if [ $(agentName) == "blobfuse-rhel7_5" ]; then + # f=`ls ./blobfuse2*$(vmImage)*.rpm` + # cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") + # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") + # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") + # cp ./blobfuse2*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) + # cp ./blobfuse2*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*RHEL-7.8*.rpm + # rm -rf ./blobfuse2*RHEL-8*.rpm + # fi + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip blobfuse2*$(vmImage)*.rpm + # sudo yum groupinstall "Development Tools" -y + # if [[ $(agentName) == "blobfuse-rhel7_5" || $(agentName) == "blobfuse-rhel7_8" ]]; then + # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y + # else + # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing + # fi + # sudo rpm -i blobfuse2*$(vmImage)*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # CentOS-7.9: + # agentName: "blobfuse-centos7" + # vmImage: 'CentOS-7.0' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-cent-7' + # CentOS-8.5: + # agentName: "blobfuse-centos8" + # vmImage: 'CentOS-8.0' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: 'test-cnt-cent-8' + + # pool: + # name: "blobfuse-centos-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + # sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + # condition: eq(variables['agentName'], 'blobfuse-centos8') + # displayName: "Update OS mirrors" + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo yum install gcc gcc-c++ make -y + # if [ $(agentName) == "blobfuse-centos8" ]; then + # sudo yum install fuse fuse3 fuse3-devel -y --nobest --allowerasing + # else + # sudo yum install fuse fuse3 fuse3-devel -y + # fi + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_6 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Oracle-8.1: + # agentName: "blobfuse-oracle81" + # vmImage: 'Oracle-8.1' + # fuselib: 'fuse3 fuse3-devel' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: "test-cnt-ora-81" + + # pool: + # name: "blobfuse-oracle-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo yum install gcc gcc-c++ make -y + # sudo yum install fuse $(fuse-version) -y --nobest --allowerasing + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # SUSE-15: + # agentName: "blobfuse-suse15" + # vmImage: 'SUSE-15Gen2' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: "test-cnt-suse-15" -# - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) - - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - - # # - job: Test_On_Ubuntu - # variables: - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py + # pool: + # name: "blobfuse-suse-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo zypper update -y + # sudo zypper -n install git + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo zypper -n install make cmake gcc gcc-c++ fuse fuse3 + # wget https://rpmfind.net/linux/opensuse/distribution/leap/15.2/repo/oss/x86_64/fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm + # sudo zypper -n --no-gpg-checks install fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm + # sudo rm fuse3-devel*.rpm + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' + + # - job: Set_8 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # fuselib: 'libfuse3-dev' + # fuse-version: 'fuse3' + # tags: 'fuse3' + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/mariner/blobfuse2*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed/mariner + + # - script: | + # sudo rpm -qip blobfuse2*.rpm + # sudo tdnf install gcc build-essential fuse3 fuse3-devel -y + # sudo rpm -i blobfuse2*.rpm + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/blobfuse2-signed/mariner + + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # TestArtifacts ends here + # - stage: ReleaseArtifacts + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') + # jobs: + # - job: ReleaseBlobfuse + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + + # steps: + # - checkout: none + + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.rpm + # displayName: 'List Artifacts' + + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - ${{ if eq(parameters.post_release, true) }}: + # # add release tags & push to github + # - task: GithubRelease@1 + # inputs: + # githubConnection: 'blobfuse-git-rel' + # repositoryName: 'Azure/azure-storage-fuse' + # action: 'edit' + + # target: '$(Build.SourceVersion)' + # tagSource: 'userSpecifiedTag' + + # title: ${{ parameters.tag }} + # tag: ${{ parameters.tag }} + + # assets: | + # $(Build.ArtifactStagingDirectory)/blobfuse2/* + # changeLogCompareToRelease: 'lastFullRelease' + # changeLogType: 'commitBased' + # isDraft: ${{ parameters.draft }} + # isPreRelease: ${{ parameters.prerelease }} + # assetUploadMode: replace + + # - ${{ if eq(parameters.publish_artifacts, true) }}: + # - stage: PublishArtifacts + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: PublishArtifacts + # timeoutInMinutes: 120 + # pool: + # vmImage: 'ubuntu-22.04' + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) \ No newline at end of file + # steps: + # - checkout: none + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'BlobFuse/Blobfuse' + # displayName: 'Connect to PMC artifact' + + # - script: | + # pip install pmc-cli + # displayName: 'Install pmc-cli' + + # - task: DownloadSecureFile@1 + # name: pmcCertificate + # displayName: 'Download pmc pem file' + # inputs: + # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + # - task: DownloadSecureFile@1 + # name: settings + # displayName: 'Download settings.toml file' + # inputs: + # secureFile: 'settings.toml' + + # - script: | + # pmc --version + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + # if [ $? -ne 0 ]; then + # exit 1 + # fi + # displayName: 'Test PMC installation' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Signed Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # cd mariner + # for f in ./blobfuse2*fuse3*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + # ls -lRt + # mv blobfuse2*.rpm .. + # cd .. + # rm -r mariner/ + # displayName: 'Rename Mariner binaries' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # displayName: 'List Artifacts' + + # - script: | + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload blobfuse2-signed + # displayName: 'Upload packages' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv + # cat ./packages.csv + # displayName: 'Fetch packages.csv' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` + # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + + # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` + # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" + + # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` + # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" + + # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` + # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" + + # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` + # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" + + # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` + # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" + + # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` + # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" + + # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` + # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" + + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Add uploaded packages to repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - script: | + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - stage: UpdateLatestVersion + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: UpdateVersion + # pool: + # vmImage: 'ubuntu-20.04' + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + + # steps: + # - checkout: none + + # - ${{ if eq(parameters.update_version, true) }}: + # - script: | + # sudo apt-get install python3 -y + # python3 --version + # displayName: 'Installing Python' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # # install blobfuse2 + # - script: | + # cd $(Build.ArtifactStagingDirectory)/blobfuse2 + # ls | grep -i ubuntu-20.04 + # sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y + # blobfuse2 version + # displayName: 'Installing blobfuse2' + + # - script: | + # wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py + # ls -l + # displayName: 'Getting Python script' + # workingDirectory: $(root_dir) + + # - script: | + # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`blobfuse2 version`" + # displayName: 'Updating version number' + # workingDirectory: $(root_dir) + From 16343cdf38bc35b24596fd53c020ee675f5fc8a6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 13 May 2024 18:44:35 +0530 Subject: [PATCH 009/513] Added signing artifacts stage --- release-pipelines.yml | 285 ++++++++++++++++++------------------------ 1 file changed, 120 insertions(+), 165 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 220293b5b..4c6488967 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1,3 +1,7 @@ +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' + parameters: - name: tag displayName: 'New Release Tag' @@ -45,18 +49,13 @@ stages: timeoutInMinutes: 360 strategy: matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - type: 'linux' - MacOS: - imageName: 'macos-11' - type: 'mac-os' - Windows: - imageName: 'windows-2019' - type: 'windows' - + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(imageName) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) steps: - task: GoTool@0 @@ -86,57 +85,14 @@ stages: displayName: 'Generate Linux ARM64' condition: eq(variables.type, 'linux') - - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - condition: eq(variables.type, 'windows') - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - condition: eq(variables.type, 'windows') - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - condition: eq(variables.type, 'windows') - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 - - script: | cp NOTICE.txt $(Build.ArtifactStagingDirectory) displayName: 'Copy NOTICE.txt' condition: eq(variables.type, 'linux') - - script: | - CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' - condition: eq(variables.type, 'mac-os') - - # uncomment below when manually releasing for m1 -# - script: | -# curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" -# displayName: 'Generate MacOS Build with ARM64' -# condition: eq(variables.type, 'mac-os') - - # cross compile to make sure nothing broke - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build - displayName: 'Test Cross-compiled MacOS Build with ARM64' - condition: eq(variables.type, 'mac-os') - - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -148,125 +104,124 @@ stages: # BuildArtifacts end here - # - stage: SignArtifacts - # dependsOn: BuildArtifacts - # condition: succeeded('BuildArtifacts') - # jobs: - # - job: SignBlobfuse - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: SignAzCopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: NightlyBlobFuse + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + displayName: 'List Artifacts' - # - script: | - # mkdir mariner && chmod 755 mariner - # cp blobfuse2-temp/*-fuse3*.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-temp/* mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # Send images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning blobfuse2' - # inputs: - # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/blobfuse2-temp' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_BLOBFUSE_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning blobfuse2 mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_BLOBFUSE_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm - # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/*.rpm - # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/blobfuse2-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/ - # displayName: 'Make Artifacts executable' + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + #chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + displayName: 'Make Artifacts executable' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/*.rpm - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + #md5sum $(Build.ArtifactStagingDirectory)/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2-signed' - # displayName: 'Publish Signed Artifacts' - # # SignArtifacts end here + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-signed' + displayName: 'Publish Signed Artifacts' + # SignArtifacts end here # - stage: TestArtifacts # dependsOn: SignArtifacts From 267a3d370e47a68d43ee3cc14408dcdce6288d92 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 13 May 2024 18:44:35 +0530 Subject: [PATCH 010/513] Added signing artifacts stage --- release-pipelines.yml | 283 ++++++++++++++++++------------------------ 1 file changed, 118 insertions(+), 165 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 220293b5b..78fa1fa02 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1,3 +1,7 @@ +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' + parameters: - name: tag displayName: 'New Release Tag' @@ -45,18 +49,13 @@ stages: timeoutInMinutes: 360 strategy: matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - type: 'linux' - MacOS: - imageName: 'macos-11' - type: 'mac-os' - Windows: - imageName: 'windows-2019' - type: 'windows' - + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(imageName) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) steps: - task: GoTool@0 @@ -86,57 +85,14 @@ stages: displayName: 'Generate Linux ARM64' condition: eq(variables.type, 'linux') - - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - condition: eq(variables.type, 'windows') - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - condition: eq(variables.type, 'windows') - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - condition: eq(variables.type, 'windows') - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 - - script: | cp NOTICE.txt $(Build.ArtifactStagingDirectory) displayName: 'Copy NOTICE.txt' condition: eq(variables.type, 'linux') - - script: | - CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' - condition: eq(variables.type, 'mac-os') - - # uncomment below when manually releasing for m1 -# - script: | -# curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL" -# displayName: 'Generate MacOS Build with ARM64' -# condition: eq(variables.type, 'mac-os') - - # cross compile to make sure nothing broke - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build - displayName: 'Test Cross-compiled MacOS Build with ARM64' - condition: eq(variables.type, 'mac-os') - - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -148,125 +104,122 @@ stages: # BuildArtifacts end here - # - stage: SignArtifacts - # dependsOn: BuildArtifacts - # condition: succeeded('BuildArtifacts') - # jobs: - # - job: SignBlobfuse - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: SignAzCopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: NightlyBlobFuse + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + displayName: 'List Artifacts' - # - script: | - # mkdir mariner && chmod 755 mariner - # cp blobfuse2-temp/*-fuse3*.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-temp/* mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # Send images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning blobfuse2' - # inputs: - # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/blobfuse2-temp' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_BLOBFUSE_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning blobfuse2 mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP Blobfuse2 Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_BLOBFUSE_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.rpm - # chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.deb - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/*.rpm - # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/blobfuse2-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2-temp/ - # displayName: 'Make Artifacts executable' + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + displayName: 'Make Artifacts executable' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/*.rpm - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2-signed' - # displayName: 'Publish Signed Artifacts' - # # SignArtifacts end here + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-signed' + displayName: 'Publish Signed Artifacts' + # SignArtifacts end here # - stage: TestArtifacts # dependsOn: SignArtifacts From e308219edaf4c6b9d4a26f7fe1280a4514478aed Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 15 May 2024 18:53:08 +0530 Subject: [PATCH 011/513] Creating rpm package --- release-pipelines.yml | 240 +++++++++++++++++++++++------------------- 1 file changed, 134 insertions(+), 106 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 78fa1fa02..5fd8f2ad0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -90,6 +90,34 @@ stages: displayName: 'Copy NOTICE.txt' condition: eq(variables.type, 'linux') + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDir/usr/bin/ + cp azure-storage-fuse/azCopy pkgDir/usr/bin/azCopy + workingDirectory: $(root_dir) + displayName: 'Accumulate pkg files' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ -v `./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3` -d $(depends) \ + --maintainer "AzCopy v-Team " --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3` -d $(depends) \ + --maintainer "Azcopy v-Team " --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' + + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' @@ -104,122 +132,122 @@ stages: # BuildArtifacts end here - - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: SignAzCopy - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - stage: SignArtifacts + # dependsOn: BuildArtifacts + # condition: succeeded('BuildArtifacts') + # jobs: + # - job: SignAzCopy + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + # displayName: 'List Artifacts' - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-temp/* mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-temp/* mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: '*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ - displayName: 'Make Artifacts executable' + # # Send images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + # Pattern: '*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + # displayName: 'Make Artifacts executable' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-signed' - displayName: 'Publish Signed Artifacts' - # SignArtifacts end here + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-signed' + # displayName: 'Publish Signed Artifacts' + # # SignArtifacts end here # - stage: TestArtifacts # dependsOn: SignArtifacts From 33f33491d978f77586e81fe3108d18db77920a02 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 15 May 2024 18:56:23 +0530 Subject: [PATCH 012/513] using devops image for testing --- release-pipelines.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5fd8f2ad0..45f9db0a9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -49,13 +49,11 @@ stages: timeoutInMinutes: 360 strategy: matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" + Ubuntu-20: + imageName: 'ubuntu-20.04' + type: 'linux' pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + vmImage: $(imageName) steps: - task: GoTool@0 From 55fa5b31f2a2565d71a6172e1afe486720106a2c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 15 May 2024 19:03:21 +0530 Subject: [PATCH 013/513] removed workingDir taf --- release-pipelines.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 45f9db0a9..f8952e9ec 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -92,8 +92,9 @@ stages: - script: | mkdir -p pkgDir/usr/bin/ cp azure-storage-fuse/azCopy pkgDir/usr/bin/azCopy - workingDirectory: $(root_dir) displayName: 'Accumulate pkg files' + #workingDirectory: $(root_dir) + # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ @@ -103,8 +104,9 @@ stages: --description "Add description" mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) displayName: 'Make deb Package' + #workingDirectory: $(root_dir) + - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3` -d $(depends) \ @@ -112,8 +114,9 @@ stages: --description "Add description" mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) displayName: 'Make rpm Package' + #workingDirectory: $(root_dir) + - task: PublishBuildArtifacts@1 From b9cc2cf3005a038b25e40eb2963836f961948d8b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 15 May 2024 19:03:21 +0530 Subject: [PATCH 014/513] removed workingDir taf --- release-pipelines.yml | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 45f9db0a9..9d8b14319 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -55,6 +55,12 @@ stages: pool: vmImage: $(imageName) + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + steps: - task: GoTool@0 env: @@ -66,45 +72,48 @@ stages: echo 'Running GO Vet' go vet displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) + workingDirectory: $root_dir - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64" + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$work_dir/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' condition: eq(variables.type, 'linux') - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$$work_dir/azcopy_linux_se_amd64" displayName: 'Generate Linux AMD64 SE Integration' condition: eq(variables.type, 'linux') - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$work_dir/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' condition: eq(variables.type, 'linux') - script: | - cp NOTICE.txt $(Build.ArtifactStagingDirectory) + cp NOTICE.txt $work_dir displayName: 'Copy NOTICE.txt' condition: eq(variables.type, 'linux') # place the generated binary files & any additional files in appropriate locations - script: | mkdir -p pkgDir/usr/bin/ - cp azure-storage-fuse/azCopy pkgDir/usr/bin/azCopy - workingDirectory: $(root_dir) + cp $work_dir pkgDir/usr/bin/azCopy displayName: 'Accumulate pkg files' + #workingDirectory: $(root_dir) + # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO check for every exe - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ -v `./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3` -d $(depends) \ + fpm -s dir -t deb -n azcopy -C pkgDir/ -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` -d $(depends) \ --maintainer "AzCopy v-Team " --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) displayName: 'Make deb Package' + #workingDirectory: $(root_dir) + - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3` -d $(depends) \ @@ -112,8 +121,9 @@ stages: --description "Add description" mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azCopy --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) displayName: 'Make rpm Package' + #workingDirectory: $(root_dir) + - task: PublishBuildArtifacts@1 From 50349a7b70e147825271b063b12652982b15c781 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 16 May 2024 17:04:45 +0530 Subject: [PATCH 015/513] creating deb package --- release-pipelines.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ea155e163..5620e656f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -119,23 +119,23 @@ stages: # TODO check for every exe - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) displayName: 'Make deb Package' #workingDirectory: $(root_dir) - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` -d $(depends) \ - --maintainer "Azcopy v-Team " --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Make rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` -d $(depends) \ + # --maintainer "Azcopy v-Team " --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Make rpm Package' #workingDirectory: $(root_dir) From 80d746fe621cbb0c58ae4efc4935b48a7f1a1d3b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 16 May 2024 17:19:40 +0530 Subject: [PATCH 016/513] creating deb package --- release-pipelines.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5620e656f..63e647e43 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -125,20 +125,19 @@ stages: --description "Add description" mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) displayName: 'Make deb Package' - #workingDirectory: $(root_dir) - - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` -d $(depends) \ - # --maintainer "Azcopy v-Team " --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # displayName: 'Make rpm Package' - #workingDirectory: $(root_dir) - - + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' - task: PublishBuildArtifacts@1 inputs: From 818127eb9f531d3cbde51f85361279efd3d59aef Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 16 May 2024 17:35:17 +0530 Subject: [PATCH 017/513] Add signing step --- release-pipelines.yml | 212 +++++++++++++++++++++--------------------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 63e647e43..364498bf5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -153,122 +153,122 @@ stages: # BuildArtifacts end here - # - stage: SignArtifacts - # dependsOn: BuildArtifacts - # condition: succeeded('BuildArtifacts') - # jobs: - # - job: SignAzCopy - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: SignAzCopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + displayName: 'List Artifacts' - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-temp/* mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-temp/* mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # Send images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - # Pattern: '*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ - # displayName: 'Make Artifacts executable' + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + displayName: 'Make Artifacts executable' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-signed' - # displayName: 'Publish Signed Artifacts' - # # SignArtifacts end here + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-signed' + displayName: 'Publish Signed Artifacts' + # SignArtifacts end here # - stage: TestArtifacts # dependsOn: SignArtifacts From 07e9a9aef77728cdb4c045d330ad19d4e94e6684 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 16 May 2024 19:13:09 +0530 Subject: [PATCH 018/513] build binaries for all the os arch --- release-pipelines.yml | 221 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 204 insertions(+), 17 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 364498bf5..02d2966cf 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -45,17 +45,21 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Build - timeoutInMinutes: 360 + - job: Linux_Amd64 + timeoutInMinutes: 120 strategy: matrix: - Ubuntu-20: + Linux_amd64: imageName: 'ubuntu-20.04' - type: 'linux' + type: 'linux_amd64' tags: 'tags' depends: 'depends' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(imageName) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) variables: - name: root_dir @@ -74,7 +78,7 @@ stages: echo 'Running GO Vet' go vet displayName: 'Golang Vet - Linux' - #workingDirectory: $root_dir + workingDirectory: $root_dir # install dependencies required for compiling azcopy - script: | @@ -85,22 +89,205 @@ stages: - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' + + - script: | + cp NOTICE.txt $(work_dir) + displayName: 'Copy NOTICE.txt' condition: eq(variables.type, 'linux') + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDir/usr/bin/ + cp -r $(work_dir) pkgDir/usr/bin/ + displayName: 'Accumulate pkg files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "AzCopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Linux_SE_Amd64 + timeoutInMinutes: 120 + strategy: + matrix: + Linux_amd64_se: + imageName: 'ubuntu-20.04' + type: 'linux_se_amd64' + tags: 'tags' + depends: 'depends' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $root_dir + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo gem install fpm -V + displayName: "Installing Dependencies" + - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" displayName: 'Generate Linux AMD64 SE Integration' - condition: eq(variables.type, 'linux') + + - script: | + cp NOTICE.txt $(work_dir) + displayName: 'Copy NOTICE.txt' + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDir/usr/bin/ + cp -r $(work_dir) pkgDir/usr/bin/ + displayName: 'Accumulate pkg files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ + --maintainer "AzCopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Linux_Arm64 + timeoutInMinutes: 120 + strategy: + matrix: + Linux_arm64: + imageName: 'ubuntu-20.04' + type: 'linux_arm64' + tags: 'tags' + depends: 'depends' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $root_dir + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo gem install fpm -V + displayName: "Installing Dependencies" - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' - condition: eq(variables.type, 'linux') - script: | cp NOTICE.txt $(work_dir) displayName: 'Copy NOTICE.txt' - condition: eq(variables.type, 'linux') # place the generated binary files & any additional files in appropriate locations - script: | @@ -108,33 +295,33 @@ stages: cp -r $(work_dir) pkgDir/usr/bin/ displayName: 'Accumulate pkg files' + # TODO: remove after testing - script: | sudo ls -lRt pkgDir/usr/bin/ displayName: 'List the binaries' - #workingDirectory: $(root_dir) + workingDirectory: $(root_dir) # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO check for every exe - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make rpm Package' @@ -214,7 +401,7 @@ stages: inputs: ConnectedServiceName: 'PMC ESRP AzCopy Signing' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: '*' + Pattern: '*.rpm, *.deb' signConfigType: inlineSignParams VerboseLogin: true inlineOperation: | @@ -233,7 +420,7 @@ stages: inputs: ConnectedServiceName: 'PMC ESRP azCopy Signing' FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*' + Pattern: '*.rpm, *.deb' signConfigType: inlineSignParams VerboseLogin: true inlineOperation: | From f104e73d50bb1e972f5495dc6023bd97fc7e775f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 16 May 2024 19:26:38 +0530 Subject: [PATCH 019/513] build binaries for all the os arch --- release-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 02d2966cf..d20ff623c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -45,7 +45,7 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Linux_Amd64 + - job: Set_1 timeoutInMinutes: 120 strategy: matrix: @@ -78,7 +78,7 @@ stages: echo 'Running GO Vet' go vet displayName: 'Golang Vet - Linux' - workingDirectory: $root_dir + workingDirectory: $(root_dir) # install dependencies required for compiling azcopy - script: | @@ -143,7 +143,7 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Linux_SE_Amd64 + - job: Set_2 timeoutInMinutes: 120 strategy: matrix: @@ -176,7 +176,7 @@ stages: echo 'Running GO Vet' go vet displayName: 'Golang Vet - Linux' - workingDirectory: $root_dir + workingDirectory: $(root_dir) # install dependencies required for compiling azcopy - script: | @@ -240,7 +240,7 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Linux_Arm64 + - job: Set_3 timeoutInMinutes: 120 strategy: matrix: @@ -273,7 +273,7 @@ stages: echo 'Running GO Vet' go vet displayName: 'Golang Vet - Linux' - workingDirectory: $root_dir + workingDirectory: $(root_dir) # install dependencies required for compiling azcopy - script: | From 67168bfdf840a8641cd065aefaf489eef82bd065 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 15:14:45 +0530 Subject: [PATCH 020/513] build binaries for all the os arch --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index d20ff623c..7abfa77f6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,6 +83,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing + sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From ff182722912156b9da778e38d539fbf51fe3dffe Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 15:25:25 +0530 Subject: [PATCH 021/513] build binaries for all the os arch --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7abfa77f6..3ed04ab52 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,7 +83,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo gem install dotenv -v 2.8.1 + sudo apt-get install gem -y sudo gem install fpm -V displayName: "Installing Dependencies" From 0c485aab5a33d586229e3d591e2ededd82dbe4d9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 16:21:06 +0530 Subject: [PATCH 022/513] build binaries for all the os arch --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 3ed04ab52..175ba2783 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,7 +83,8 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install gem -y + sudo apt-get install ruby-dev + sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From 354448b595114b88d74f711409ba56198c2755a3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 16:45:04 +0530 Subject: [PATCH 023/513] build binaries for all the os arch --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 175ba2783..518fab54c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -85,7 +85,7 @@ stages: sudo apt-get update --fix-missing sudo apt-get install ruby-dev sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V + gem install fpm -V displayName: "Installing Dependencies" - script: | From d73c917f9870240b287875481faf322e8154cbca Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 16:55:10 +0530 Subject: [PATCH 024/513] build binaries for all the os arch --- release-pipelines.yml | 188 +++++++++++++++++++++--------------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 518fab54c..5e8ef8498 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -85,7 +85,7 @@ stages: sudo apt-get update --fix-missing sudo apt-get install ruby-dev sudo gem install dotenv -v 2.8.1 - gem install fpm -V + sudo gem install fpm -V displayName: "Installing Dependencies" - script: | @@ -145,104 +145,104 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Linux_amd64_se: - imageName: 'ubuntu-20.04' - type: 'linux_se_amd64' - tags: 'tags' - depends: 'depends' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo gem install fpm -V - displayName: "Installing Dependencies" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - cp NOTICE.txt $(work_dir) - displayName: 'Copy NOTICE.txt' - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - cp -r $(work_dir) pkgDir/usr/bin/ - displayName: 'Accumulate pkg files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux_amd64_se: + # imageName: 'ubuntu-20.04' + # type: 'linux_se_amd64' + # tags: 'tags' + # depends: 'depends' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # cp NOTICE.txt $(work_dir) + # displayName: 'Copy NOTICE.txt' + + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp -r $(work_dir) pkgDir/usr/bin/ + # displayName: 'Accumulate pkg files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ + # --maintainer "AzCopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make rpm Package' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make rpm Package' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_3 + # - job: Set_3 timeoutInMinutes: 120 strategy: matrix: From 37da016eb5afd9ae026b64f19059edf537dc1416 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 16:56:31 +0530 Subject: [PATCH 025/513] build binaries for all the os arch --- release-pipelines.yml | 166 +++++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5e8ef8498..c4a8b7778 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -243,102 +243,102 @@ stages: # alertWarningLevel: 'High' # - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Linux_arm64: - imageName: 'ubuntu-20.04' - type: 'linux_arm64' - tags: 'tags' - depends: 'depends' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux_arm64: + # imageName: 'ubuntu-20.04' + # type: 'linux_arm64' + # tags: 'tags' + # depends: 'depends' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo gem install fpm -V - displayName: "Installing Dependencies" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo gem install fpm -V + # displayName: "Installing Dependencies" - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - cp NOTICE.txt $(work_dir) - displayName: 'Copy NOTICE.txt' + # - script: | + # cp NOTICE.txt $(work_dir) + # displayName: 'Copy NOTICE.txt' - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - cp -r $(work_dir) pkgDir/usr/bin/ - displayName: 'Accumulate pkg files' + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp -r $(work_dir) pkgDir/usr/bin/ + # displayName: 'Accumulate pkg files' - # TODO: remove after testing - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) + # # TODO: remove after testing + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "AzCopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make rpm Package' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # BuildArtifacts end here From 13d3c757c2d1d576c081866db76d35f3238ff933 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 17:08:56 +0530 Subject: [PATCH 026/513] build binaries for all the os arch --- release-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c4a8b7778..1dc823b7f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,7 +83,6 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install ruby-dev sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From 2261d4d2bb8fdd902f687cd7722932e73b02d256 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 17:15:46 +0530 Subject: [PATCH 027/513] build binaries for all the os arch --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1dc823b7f..8043ee200 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,6 +83,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing + sudo apt-get install rubygems sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From 24f79b8f9bf99fad4ef5ff016400b2123905ca60 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 17:22:29 +0530 Subject: [PATCH 028/513] build binaries for all the os arch --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8043ee200..cd1be2e44 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,7 +83,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install rubygems + sudo apt-get install rubygems ruby-dev sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From 9181bbcd3cf7eef489e2127b8a785c48fc5091f3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 17:23:36 +0530 Subject: [PATCH 029/513] build binaries for all the os arch --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index cd1be2e44..77ebe7af7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,7 +83,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install rubygems ruby-dev + apt-get install ruby ruby-dev rubygems build-essential sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From 841b1404ad996b52a9e0aa25fd6df00ffe71f549 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 17:29:19 +0530 Subject: [PATCH 030/513] build binaries for all the os arch --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 77ebe7af7..e32567ba8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -83,7 +83,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - apt-get install ruby ruby-dev rubygems build-essential + sudo apt-get install ruby ruby-dev rubygems build-essential sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From a1fb823b6c67db5dae45f9882aec325b3bb75188 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 18:38:15 +0530 Subject: [PATCH 031/513] build binaries for all the os arch --- release-pipelines.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e32567ba8..d77e7861e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -49,17 +49,23 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Linux_amd64: + Ubuntu-20: imageName: 'ubuntu-20.04' - type: 'linux_amd64' - tags: 'tags' - depends: 'depends' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + type: 'linux' + # Linux_amd64: + # imageName: 'ubuntu-20.04' + # type: 'linux_amd64' + # tags: 'tags' + # depends: 'depends' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) + vmImage: $(imageName) variables: - name: root_dir @@ -83,7 +89,6 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems build-essential sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From 5dfdcbb582d3f9990f9f19dbce1b04a9486b543c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 18:45:25 +0530 Subject: [PATCH 032/513] build binaries for all the os arch --- release-pipelines.yml | 384 ++++++++++++++++++++++-------------------- 1 file changed, 198 insertions(+), 186 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d77e7861e..f9a67dcd8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -150,200 +150,212 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux_amd64_se: - # imageName: 'ubuntu-20.04' - # type: 'linux_se_amd64' - # tags: 'tags' - # depends: 'depends' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get update --fix-missing - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # cp NOTICE.txt $(work_dir) - # displayName: 'Copy NOTICE.txt' - - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp -r $(work_dir) pkgDir/usr/bin/ - # displayName: 'Accumulate pkg files' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + type: 'linux' + # Linux_amd64_se: + # imageName: 'ubuntu-20.04' + # type: 'linux_se_amd64' + # tags: 'tags' + # depends: 'depends' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo gem install fpm -V + displayName: "Installing Dependencies" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + cp NOTICE.txt $(work_dir) + displayName: 'Copy NOTICE.txt' + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDir/usr/bin/ + cp -r $(work_dir) pkgDir/usr/bin/ + displayName: 'Accumulate pkg files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ - # --maintainer "AzCopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ + --maintainer "AzCopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make rpm Package' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux_arm64: - # imageName: 'ubuntu-20.04' - # type: 'linux_arm64' - # tags: 'tags' - # depends: 'depends' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get update --fix-missing - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' - - # - script: | - # cp NOTICE.txt $(work_dir) - # displayName: 'Copy NOTICE.txt' - - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp -r $(work_dir) pkgDir/usr/bin/ - # displayName: 'Accumulate pkg files' - - # # TODO: remove after testing - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + type: 'linux' + # Linux_arm64: + # imageName: 'ubuntu-20.04' + # type: 'linux_arm64' + # tags: 'tags' + # depends: 'depends' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo gem install fpm -V + displayName: "Installing Dependencies" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + cp NOTICE.txt $(work_dir) + displayName: 'Copy NOTICE.txt' + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDir/usr/bin/ + cp -r $(work_dir) pkgDir/usr/bin/ + displayName: 'Accumulate pkg files' + + # TODO: remove after testing + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "AzCopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "AzCopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make rpm Package' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # BuildArtifacts end here From 1d71a827b2c8c1b9c7a17cca75f766ffb6a9eba8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 19:14:26 +0530 Subject: [PATCH 033/513] using blobfuse-pool --- release-pipelines.yml | 327 +++++++++++++++--------------------------- 1 file changed, 112 insertions(+), 215 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f9a67dcd8..4a2e5ad5d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -49,23 +49,23 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - type: 'linux' - # Linux_amd64: + # Ubuntu-20: # imageName: 'ubuntu-20.04' - # type: 'linux_amd64' - # tags: 'tags' - # depends: 'depends' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - + # type: 'linux' + Linux_amd64: + imageName: 'ubuntu-20.04' + type: 'linux_amd64' + tags: 'tags' + depends: 'depends' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(imageName) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + # pool: + # vmImage: $(imageName) variables: - name: root_dir @@ -89,6 +89,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing + sudo apt-get install ruby ruby-dev rubygems build-essential -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" @@ -116,6 +117,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ @@ -150,109 +152,6 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - type: 'linux' - # Linux_amd64_se: - # imageName: 'ubuntu-20.04' - # type: 'linux_se_amd64' - # tags: 'tags' - # depends: 'depends' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo gem install fpm -V - displayName: "Installing Dependencies" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - cp NOTICE.txt $(work_dir) - displayName: 'Copy NOTICE.txt' - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - cp -r $(work_dir) pkgDir/usr/bin/ - displayName: 'Accumulate pkg files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make deb Package' - - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_se_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make rpm Package' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - job: Set_3 timeoutInMinutes: 120 strategy: @@ -476,112 +375,110 @@ stages: displayName: 'Publish Signed Artifacts' # SignArtifacts end here - # - stage: TestArtifacts - # dependsOn: SignArtifacts - # condition: succeeded('SignArtifacts') - # jobs: - # - job: Set_0 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # fuse-version: 'fuse3' - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + tags: 'fuse3' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/azcopy-temp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-fuse - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'blobfuse2-signed' + downloadPath: $(root_dir) + itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # sudo dpkg --info blobfuse2*.deb - # sudo dpkg -i blobfuse2*.deb - # sudo apt-get install $(fuse-version) build-essential -y - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # Create container for blobfuse and publish it - # - script: | - # sudo apt-get install $(tags) $(fuselib) -y - # chmod 777 *.sh - # ./dockerinstall.sh - # ./buildcontainer.sh Dockerfile x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - # ./buildcontainer.sh DockerfileMariner mariner-x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - # displayName: "Create and publish container" - # workingDirectory: $(work_dir)/docker - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + - script: | + sudo dpkg --info blobfuse2*.deb + sudo dpkg -i blobfuse2*.deb + sudo apt-get install $(fuse-version) build-essential -y + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + - template: 'azure-pipeline-templates/release-distro-tests.yml' + parameters: + root_dir: $(root_dir) + work_dir: $(work_dir) + mount_dir: $(mount_dir) + temp_dir: $(temp_dir) + container: $(container) + + # Create container for blobfuse and publish it + - script: | + sudo apt-get install $(tags) $(fuselib) -y + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh Dockerfile x86_64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + ./buildcontainer.sh DockerfileMariner mariner-x86_64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + displayName: "Create and publish container" + workingDirectory: $(work_dir)/docker + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'blobfuse2' + displayName: 'Publish Artifacts' # - job: Set_1 # timeoutInMinutes: 120 From 029d5083da3ea19d06379378b5f85eb57de09937 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 19:15:41 +0530 Subject: [PATCH 034/513] using blobfuse-pool --- release-pipelines.yml | 190 +++++++++++++++++++++--------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4a2e5ad5d..a04127a40 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -375,110 +375,110 @@ stages: displayName: 'Publish Signed Artifacts' # SignArtifacts end here - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - tags: 'fuse3' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # tags: 'fuse3' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) - variables: - - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/azcopy-temp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/azcopy-temp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed - - script: | - sudo dpkg --info blobfuse2*.deb - sudo dpkg -i blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # Create container for blobfuse and publish it - - script: | - sudo apt-get install $(tags) $(fuselib) -y - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh Dockerfile x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - ./buildcontainer.sh DockerfileMariner mariner-x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - displayName: "Create and publish container" - workingDirectory: $(work_dir)/docker - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo dpkg -i blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) + + # # Create container for blobfuse and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh Dockerfile x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + # ./buildcontainer.sh DockerfileMariner mariner-x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + # displayName: "Create and publish container" + # workingDirectory: $(work_dir)/docker + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' # - job: Set_1 # timeoutInMinutes: 120 From 4abde581512e94f5a3aedb56ef9636350714d2cb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 19:25:06 +0530 Subject: [PATCH 035/513] using blobfuse-pool --- release-pipelines.yml | 433 +++++++++++++++++++++--------------------- 1 file changed, 215 insertions(+), 218 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a04127a40..8b1712aca 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -89,7 +89,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems build-essential -y + sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" @@ -152,27 +152,22 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 + - job: Set_2 timeoutInMinutes: 120 strategy: matrix: - Ubuntu-20: + Linux_arm64: imageName: 'ubuntu-20.04' - type: 'linux' - # Linux_arm64: - # imageName: 'ubuntu-20.04' - # type: 'linux_arm64' - # tags: 'tags' - # depends: 'depends' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) + type: 'linux_arm64' + tags: 'tags' + depends: 'depends' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(imageName) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) variables: - name: root_dir @@ -196,6 +191,8 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing + sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y + sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" @@ -375,230 +372,230 @@ stages: displayName: 'Publish Signed Artifacts' # SignArtifacts end here - # - stage: TestArtifacts - # dependsOn: SignArtifacts - # condition: succeeded('SignArtifacts') - # jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # tags: 'fuse3' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + tags: 'fuse3' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/azcopy-temp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/azcopy-temp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-fuse - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'blobfuse2-signed' + downloadPath: $(root_dir) + itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # sudo dpkg --info blobfuse2*.deb - # sudo dpkg -i blobfuse2*.deb - # sudo apt-get install $(fuse-version) build-essential -y - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # Create container for blobfuse and publish it - # - script: | - # sudo apt-get install $(tags) $(fuselib) -y - # chmod 777 *.sh - # ./dockerinstall.sh - # ./buildcontainer.sh Dockerfile x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - # ./buildcontainer.sh DockerfileMariner mariner-x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - # displayName: "Create and publish container" - # workingDirectory: $(work_dir)/docker - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + - script: | + sudo dpkg --info blobfuse2*.deb + sudo dpkg -i blobfuse2*.deb + sudo apt-get install $(fuse-version) build-essential -y + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + - template: 'azure-pipeline-templates/release-distro-tests.yml' + parameters: + root_dir: $(root_dir) + work_dir: $(work_dir) + mount_dir: $(mount_dir) + temp_dir: $(temp_dir) + container: $(container) + + # Create container for blobfuse and publish it + - script: | + sudo apt-get install $(tags) $(fuselib) -y + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh Dockerfile x86_64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + ./buildcontainer.sh DockerfileMariner mariner-x86_64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + displayName: "Create and publish container" + workingDirectory: $(work_dir)/docker + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'blobfuse2' + displayName: 'Publish Artifacts' - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-18: - # vmImage: 'Ubuntu-18.04' - # fuselib: 'libfuse-dev' - # fuse-version: 'fuse' - # tags: 'fuse2' - # container: 'test-cnt-ubn-18' - # AgentName: "blobfuse-ubuntu18" - # Ubuntu-20: - # vmImage: 'Ubuntu-20.04' - # fuse-version: 'fuse3' - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-18: + vmImage: 'Ubuntu-18.04' + fuselib: 'libfuse-dev' + fuse-version: 'fuse' + tags: 'fuse2' + container: 'test-cnt-ubn-18' + AgentName: "blobfuse-ubuntu18" + Ubuntu-20: + vmImage: 'Ubuntu-20.04' + fuse-version: 'fuse3' + fuselib: 'libfuse3-dev' + tags: 'fuse3' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + variables: + - group: NightlyBlobFuse + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/fusetmp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-fuse - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'blobfuse2-signed' + downloadPath: $(root_dir) + itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # if [ $(AgentName) == "blobfuse-ubuntu18" ]; then - # echo "Generating for Debian 9/10" - # f=`ls ./blobfuse2*$(vmImage)*.deb` - # cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") - # cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") - # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - # ls -l $(Build.ArtifactStagingDirectory) - # rm -rf ./blobfuse2*Debian-*.deb - # else - # echo "Generating for Debian 11" - # f=`ls ./blobfuse2*$(vmImage)*.deb` - # cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") - # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - # ls -l $(Build.ArtifactStagingDirectory) - # rm -rf ./blobfuse2*Debian-*.deb - # fi - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + if [ $(AgentName) == "blobfuse-ubuntu18" ]; then + echo "Generating for Debian 9/10" + f=`ls ./blobfuse2*$(vmImage)*.deb` + cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") + cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") + cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + ls -l $(Build.ArtifactStagingDirectory) + rm -rf ./blobfuse2*Debian-*.deb + else + echo "Generating for Debian 11" + f=`ls ./blobfuse2*$(vmImage)*.deb` + cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") + cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + ls -l $(Build.ArtifactStagingDirectory) + rm -rf ./blobfuse2*Debian-*.deb + fi + displayName: 'Rename Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # sudo dpkg --info blobfuse2*.deb - # sudo apt-get install $(fuse-version) build-essential -y - # sudo dpkg -i blobfuse2*.deb - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo dpkg --info blobfuse2*.deb + sudo apt-get install $(fuse-version) build-essential -y + sudo dpkg -i blobfuse2*.deb + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) + - template: 'azure-pipeline-templates/release-distro-tests.yml' + parameters: + root_dir: $(root_dir) + work_dir: $(work_dir) + mount_dir: $(mount_dir) + temp_dir: $(temp_dir) + container: $(container) - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'blobfuse2' + displayName: 'Publish Artifacts' # - job: Set_2_ARM # timeoutInMinutes: 120 From 8b560e063f26f4368bf6d56c5714729679c388f4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 19:26:01 +0530 Subject: [PATCH 036/513] using blobfuse-pool --- release-pipelines.yml | 404 +++++++++++++++++++++--------------------- 1 file changed, 202 insertions(+), 202 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8b1712aca..5899828cd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -372,230 +372,230 @@ stages: displayName: 'Publish Signed Artifacts' # SignArtifacts end here - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - tags: 'fuse3' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # tags: 'fuse3' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) - variables: - - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/azcopy-temp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/azcopy-temp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed - - script: | - sudo dpkg --info blobfuse2*.deb - sudo dpkg -i blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # Create container for blobfuse and publish it - - script: | - sudo apt-get install $(tags) $(fuselib) -y - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh Dockerfile x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - ./buildcontainer.sh DockerfileMariner mariner-x86_64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - displayName: "Create and publish container" - workingDirectory: $(work_dir)/docker - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo dpkg -i blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-18: - vmImage: 'Ubuntu-18.04' - fuselib: 'libfuse-dev' - fuse-version: 'fuse' - tags: 'fuse2' - container: 'test-cnt-ubn-18' - AgentName: "blobfuse-ubuntu18" - Ubuntu-20: - vmImage: 'Ubuntu-20.04' - fuse-version: 'fuse3' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # # Create container for blobfuse and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh Dockerfile x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + # ./buildcontainer.sh DockerfileMariner mariner-x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + # displayName: "Create and publish container" + # workingDirectory: $(work_dir)/docker - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' - steps: - - checkout: none + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-18: + # vmImage: 'Ubuntu-18.04' + # fuselib: 'libfuse-dev' + # fuse-version: 'fuse' + # tags: 'fuse2' + # container: 'test-cnt-ubn-18' + # AgentName: "blobfuse-ubuntu18" + # Ubuntu-20: + # vmImage: 'Ubuntu-20.04' + # fuse-version: 'fuse3' + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + # variables: + # - group: NightlyBlobFuse + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + + # steps: + # - checkout: none + + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-fuse - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/blobfuse2-signed - - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - if [ $(AgentName) == "blobfuse-ubuntu18" ]; then - echo "Generating for Debian 9/10" - f=`ls ./blobfuse2*$(vmImage)*.deb` - cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") - cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") - cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - ls -l $(Build.ArtifactStagingDirectory) - rm -rf ./blobfuse2*Debian-*.deb - else - echo "Generating for Debian 11" - f=`ls ./blobfuse2*$(vmImage)*.deb` - cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") - cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - ls -l $(Build.ArtifactStagingDirectory) - rm -rf ./blobfuse2*Debian-*.deb - fi - displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed + # - script: | + # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # if [ $(AgentName) == "blobfuse-ubuntu18" ]; then + # echo "Generating for Debian 9/10" + # f=`ls ./blobfuse2*$(vmImage)*.deb` + # cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") + # cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") + # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + # ls -l $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*Debian-*.deb + # else + # echo "Generating for Debian 11" + # f=`ls ./blobfuse2*$(vmImage)*.deb` + # cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") + # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) + # ls -l $(Build.ArtifactStagingDirectory) + # rm -rf ./blobfuse2*Debian-*.deb + # fi + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/blobfuse2-signed - - script: | - sudo dpkg --info blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - sudo dpkg -i blobfuse2*.deb - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo dpkg --info blobfuse2*.deb + # sudo apt-get install $(fuse-version) build-essential -y + # sudo dpkg -i blobfuse2*.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' # - job: Set_2_ARM # timeoutInMinutes: 120 From b722accf9792613532b91aa8a9b2f5f101453fba Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 17 May 2024 19:40:31 +0530 Subject: [PATCH 037/513] using blobfuse arm pool image for amd64 --- release-pipelines.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5899828cd..c2eb7c5c0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -157,15 +157,12 @@ stages: strategy: matrix: Linux_arm64: - imageName: 'ubuntu-20.04' - type: 'linux_arm64' - tags: 'tags' - depends: 'depends' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + vmImage: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' pool: - name: "blobfuse-ubuntu-pool" + name: "blobfuse-ubn-arm64-pool" demands: - ImageOverride -equals $(agentName) From 4d98af2ea4c8cc3cab0a0c1b0c407b5e8eb1ac55 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 12:17:26 +0530 Subject: [PATCH 038/513] using blobfuse arm pool image for amd64 --- release-pipelines.yml | 186 +++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c2eb7c5c0..ad3db80fb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -152,103 +152,103 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Linux_arm64: - vmImage: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' - - - script: | - cp NOTICE.txt $(work_dir) - displayName: 'Copy NOTICE.txt' - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - cp -r $(work_dir) pkgDir/usr/bin/ - displayName: 'Accumulate pkg files' - - # TODO: remove after testing - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux_arm64: + # vmImage: 'ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: 'blobfuse-ubn22-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' + + # - script: | + # cp NOTICE.txt $(work_dir) + # displayName: 'Copy NOTICE.txt' + + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp -r $(work_dir) pkgDir/usr/bin/ + # displayName: 'Accumulate pkg files' + + # # TODO: remove after testing + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "AzCopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make rpm Package' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make rpm Package' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # BuildArtifacts end here From 8e00fc992d95a62a48d19a707057f7eeff35368b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 15:09:43 +0530 Subject: [PATCH 039/513] test artifacts --- release-pipelines.yml | 189 +++++++++++++++++++++--------------------- 1 file changed, 94 insertions(+), 95 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ad3db80fb..8d56369ef 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -369,110 +369,109 @@ stages: displayName: 'Publish Signed Artifacts' # SignArtifacts end here - # - stage: TestArtifacts - # dependsOn: SignArtifacts - # condition: succeeded('SignArtifacts') - # jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # tags: 'fuse3' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/azcopy-temp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/azcopy-temp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/azcopy*$(tags)*x86_64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed - # - script: | - # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + for f in ./azcopy*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed - # - script: | - # sudo dpkg --info blobfuse2*.deb - # sudo dpkg -i blobfuse2*.deb - # sudo apt-get install $(fuse-version) build-essential -y - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # Create container for blobfuse and publish it - # - script: | - # sudo apt-get install $(tags) $(fuselib) -y - # chmod 777 *.sh - # ./dockerinstall.sh - # ./buildcontainer.sh Dockerfile x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - # ./buildcontainer.sh DockerfileMariner mariner-x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - # displayName: "Create and publish container" - # workingDirectory: $(work_dir)/docker - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + - script: | + sudo dpkg --info azcopy*.deb + sudo dpkg -i azcopy*.deb + sudo apt-get install $(fuse-version) build-essential -y + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + - template: 'azure-pipeline-templates/release-distro-tests.yml' + parameters: + root_dir: $(root_dir) + work_dir: $(work_dir) + mount_dir: $(mount_dir) + temp_dir: $(temp_dir) + container: $(container) + + - script: | + azcopy --version + displayName: 'Check Version' + + # # Create container for azcopy and publish it + # - script: | + # sudo apt-get install $(tags) $(fuselib) -y + # chmod 777 *.sh + # ./dockerinstall.sh + # ./buildcontainer.sh Dockerfile x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 + # ./buildcontainer.sh DockerfileMariner mariner-x86_64 + # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 + # displayName: "Create and publish container" + # workingDirectory: $(work_dir)/docker + + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' # - job: Set_1 # timeoutInMinutes: 120 From d2b63a42d4e0d907970692c88c5b2dd252ca4c7c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 15:10:53 +0530 Subject: [PATCH 040/513] test artifacts --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8d56369ef..02f94e923 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -452,8 +452,8 @@ stages: container: $(container) - script: | - azcopy --version - displayName: 'Check Version' + azcopy --version + displayName: 'Check Version' # # Create container for azcopy and publish it # - script: | From 0a65197113f796392aa2c03538087de05f466076 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 15:11:33 +0530 Subject: [PATCH 041/513] test artifacts --- release-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 02f94e923..98504dc49 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -443,13 +443,13 @@ stages: sudo apt-get install $(fuse-version) build-essential -y displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) + # - template: 'azure-pipeline-templates/release-distro-tests.yml' + # parameters: + # root_dir: $(root_dir) + # work_dir: $(work_dir) + # mount_dir: $(mount_dir) + # temp_dir: $(temp_dir) + # container: $(container) - script: | azcopy --version From e38868c65d4f77ede126bdc0b63fc43f739b9248 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 15:27:11 +0530 Subject: [PATCH 042/513] test artifacts --- release-pipelines.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 98504dc49..b8886e892 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -408,11 +408,16 @@ stages: workingDirectory: $(root_dir)/azure-storage-azcopy # Custom script to install Go-lang - - task: ShellScript@2 + - task: GoTool@0 + env: + GO111MODULE: 'on' inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + version: $(AZCOPY_GOLANG_VERSION) + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - task: DownloadBuildArtifacts@0 From 06fd7de7581738493263b495dbc0c4728ca7c108 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 15:28:37 +0530 Subject: [PATCH 043/513] test artifacts --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index b8886e892..549e94809 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -413,6 +413,7 @@ stages: GO111MODULE: 'on' inputs: version: $(AZCOPY_GOLANG_VERSION) + displayName: "Install Go" # - task: ShellScript@2 # inputs: # scriptPath: "$(work_dir)/go_installer.sh" From ed5ae21d4fa85b1feca2a33c2e81cf55b98d4d55 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 15:50:40 +0530 Subject: [PATCH 044/513] test artifacts --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 549e94809..dd83bb783 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -435,13 +435,13 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + #workingDirectory: $(root_dir)/azCopy-signed - script: | for f in ./azcopy*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; cp ./azcopy*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed + #workingDirectory: $(root_dir)/azCopy-signed - script: | sudo dpkg --info azcopy*.deb From de37b3edf3fbe4b6e34fa904db1c9c8cb59b38da Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 17:22:00 +0530 Subject: [PATCH 045/513] test artifacts --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index dd83bb783..b06e21eaa 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -381,6 +381,7 @@ stages: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' + tags: 'tags' pool: vmImage: $(vmImage) From 6e4e386c75ffd926cb7944b9b5147100b98dbe7a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 20 May 2024 18:02:34 +0530 Subject: [PATCH 046/513] test artifacts --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b06e21eaa..c88c061b8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -436,13 +436,13 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - #workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-signed - script: | for f in ./azcopy*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; cp ./azcopy*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' - #workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-signed - script: | sudo dpkg --info azcopy*.deb From 0f97b114a9aec04c9923fcbbe17ca5a7c2da3ccd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 16:10:24 +0530 Subject: [PATCH 047/513] add build job for arm64 --- release-pipelines.yml | 1189 +++++++++++------------------------------ 1 file changed, 304 insertions(+), 885 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c88c061b8..0a6d4c509 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -49,9 +49,6 @@ stages: timeoutInMinutes: 120 strategy: matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # type: 'linux' Linux_amd64: imageName: 'ubuntu-20.04' type: 'linux_amd64' @@ -62,10 +59,7 @@ stages: pool: name: "blobfuse-ubuntu-pool" demands: - - ImageOverride -equals $(agentName) - - # pool: - # vmImage: $(imageName) + - ImageOverride -equals $(agentName) variables: - name: root_dir @@ -152,103 +146,103 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux_arm64: - # vmImage: 'ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: 'blobfuse-ubn22-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get update --fix-missing - # sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' - - # - script: | - # cp NOTICE.txt $(work_dir) - # displayName: 'Copy NOTICE.txt' - - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp -r $(work_dir) pkgDir/usr/bin/ - # displayName: 'Accumulate pkg files' - - # # TODO: remove after testing - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Linux_arm64: + vmImage: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + cp NOTICE.txt $(work_dir) + displayName: 'Copy NOTICE.txt' + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDir/usr/bin/ + cp -r $(work_dir) pkgDir/usr/bin/ + displayName: 'Accumulate pkg files' + + # TODO: remove after testing + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "AzCopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "AzCopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make rpm Package' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make rpm Package' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # BuildArtifacts end here @@ -415,12 +409,6 @@ stages: inputs: version: $(AZCOPY_GOLANG_VERSION) displayName: "Install Go" - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' @@ -447,7 +435,7 @@ stages: - script: | sudo dpkg --info azcopy*.deb sudo dpkg -i azcopy*.deb - sudo apt-get install $(fuse-version) build-essential -y + sudo apt-get install build-essential -y displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) # - template: 'azure-pipeline-templates/release-distro-tests.yml' @@ -462,17 +450,9 @@ stages: azcopy --version displayName: 'Check Version' - # # Create container for azcopy and publish it - # - script: | - # sudo apt-get install $(tags) $(fuselib) -y - # chmod 777 *.sh - # ./dockerinstall.sh - # ./buildcontainer.sh Dockerfile x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) x86_64 - # ./buildcontainer.sh DockerfileMariner mariner-x86_64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) mariner-x86_64 - # displayName: "Create and publish container" - # workingDirectory: $(work_dir)/docker + - script: | + azcopy --help + displayName: 'Check Help' # # publishing the artifacts generated # - task: PublishBuildArtifacts@1 @@ -707,139 +687,23 @@ stages: # artifactName: 'blobfuse2' # displayName: 'Publish Artifacts' - # - job: Set_2_AARCH + # - job: Set_8 # timeoutInMinutes: 120 # strategy: # matrix: - # RHEL-9.0: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # fuselib: 'fuse fuse3-libs fuse3-devel' + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # fuselib: 'libfuse3-dev' # fuse-version: 'fuse3' # tags: 'fuse3' - # container: 'test-cnt-rhel-9-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - # steps: - # - checkout: none - - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*aarch64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip blobfuse2*$(vmImage)*.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing - # sudo rpm -i blobfuse2*$(vmImage)*.rpm - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) + # container: "test-cnt-mari-2" - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # RHEL-7.5: - # agentName: "blobfuse-rhel7_5" - # vmImage: 'RHEL-7.5' - # fuselib: 'fuse3-devel' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: 'test-cnt-rhel-75' - # RHEL-8.6: - # agentName: "blobfuse-rhel8_6" - # vmImage: 'RHEL-8.6' - # fuselib: 'fuse fuse3-libs fuse3-devel' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: 'test-cnt-rhel-86' - # RHEL-9.0: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # fuselib: 'fuse fuse3-libs fuse3-devel' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" + # name: "blobfuse-mariner-pool" # demands: - # - ImageOverride -equals $(agentName) + # - ImageOverride -equals $(AgentName) # variables: # - group: NightlyBlobFuse @@ -856,14 +720,8 @@ stages: # - checkout: none # - script: | - # sudo touch /etc/yum.repos.d/centos.repo - # sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' - # condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) - # displayName: "Update OS mirrors" - - # - script: | - # sudo yum update -y - # sudo yum install git -y + # sudo tdnf update -y + # sudo tdnf install git -y # displayName: 'Install Git' # - script: | @@ -893,7 +751,7 @@ stages: # inputs: # artifactName: 'blobfuse2-signed' # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm + # itemPattern: blobfuse2-signed/mariner/blobfuse2*$(tags)*x86_64.rpm # - script: | # ls -l @@ -902,36 +760,14 @@ stages: # exit 1 # fi # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # if [ $(agentName) == "blobfuse-rhel7_5" ]; then - # f=`ls ./blobfuse2*$(vmImage)*.rpm` - # cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") - # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") - # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") - # cp ./blobfuse2*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) - # cp ./blobfuse2*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) - # rm -rf ./blobfuse2*RHEL-7.8*.rpm - # rm -rf ./blobfuse2*RHEL-8*.rpm - # fi - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + # workingDirectory: $(root_dir)/blobfuse2-signed/mariner # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip blobfuse2*$(vmImage)*.rpm - # sudo yum groupinstall "Development Tools" -y - # if [[ $(agentName) == "blobfuse-rhel7_5" || $(agentName) == "blobfuse-rhel7_8" ]]; then - # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y - # else - # sudo yum install fuse fuse3-libs fuse3-devel fuse3 -y --nobest --allowerasing - # fi - # sudo rpm -i blobfuse2*$(vmImage)*.rpm + # sudo rpm -qip blobfuse2*.rpm + # sudo tdnf install gcc build-essential fuse3 fuse3-devel -y + # sudo rpm -i blobfuse2*.rpm # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir)/blobfuse2-signed/mariner # - template: 'azure-pipeline-templates/release-distro-tests.yml' # parameters: @@ -941,617 +777,200 @@ stages: # temp_dir: $(temp_dir) # container: $(container) - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + # # TestArtifacts ends here + + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseBlobfuse + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # CentOS-7.9: - # agentName: "blobfuse-centos7" - # vmImage: 'CentOS-7.0' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: 'test-cnt-cent-7' - # CentOS-8.5: - # agentName: "blobfuse-centos8" - # vmImage: 'CentOS-8.0' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: 'test-cnt-cent-8' - - # pool: - # name: "blobfuse-centos-pool" - # demands: - # - ImageOverride -equals $(agentName) + variables: + - group: AZCOPY_ESRP_SECRET - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azcopy' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + displayName: 'List Artifacts' - # steps: - # - checkout: none - - # - script: | - # sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - # sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - # condition: eq(variables['agentName'], 'blobfuse-centos8') - # displayName: "Update OS mirrors" - - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # sudo rpm -qip blobfuse2*.rpm - # sudo yum install gcc gcc-c++ make -y - # if [ $(agentName) == "blobfuse-centos8" ]; then - # sudo yum install fuse fuse3 fuse3-devel -y --nobest --allowerasing - # else - # sudo yum install fuse fuse3 fuse3-devel -y - # fi - # sudo rpm -i blobfuse2*.rpm - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' - - # - job: Set_6 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Oracle-8.1: - # agentName: "blobfuse-oracle81" - # vmImage: 'Oracle-8.1' - # fuselib: 'fuse3 fuse3-devel' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: "test-cnt-ora-81" - - # pool: - # name: "blobfuse-oracle-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - # steps: - # - checkout: none - - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # sudo rpm -qip blobfuse2*.rpm - # sudo yum install gcc gcc-c++ make -y - # sudo yum install fuse $(fuse-version) -y --nobest --allowerasing - # sudo rpm -i blobfuse2*.rpm - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' - - # - job: Set_7 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # SUSE-15: - # agentName: "blobfuse-suse15" - # vmImage: 'SUSE-15Gen2' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: "test-cnt-suse-15" - - # pool: - # name: "blobfuse-suse-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - # steps: - # - checkout: none - - # - script: | - # sudo zypper update -y - # sudo zypper -n install git - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # for f in ./blobfuse2*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # sudo rpm -qip blobfuse2*.rpm - # sudo zypper -n install make cmake gcc gcc-c++ fuse fuse3 - # wget https://rpmfind.net/linux/opensuse/distribution/leap/15.2/repo/oss/x86_64/fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm - # sudo zypper -n --no-gpg-checks install fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm - # sudo rm fuse3-devel*.rpm - # sudo rpm -i blobfuse2*.rpm - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' - - # - job: Set_8 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # fuselib: 'libfuse3-dev' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - # steps: - # - checkout: none - - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/mariner/blobfuse2*$(tags)*x86_64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed/mariner - - # - script: | - # sudo rpm -qip blobfuse2*.rpm - # sudo tdnf install gcc build-essential fuse3 fuse3-devel -y - # sudo rpm -i blobfuse2*.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/blobfuse2-signed/mariner - - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # TestArtifacts ends here - # - stage: ReleaseArtifacts - # dependsOn: TestArtifacts - # condition: succeeded('TestArtifacts') - # jobs: - # - job: ReleaseBlobfuse - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - - # steps: - # - checkout: none - - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.rpm - # displayName: 'List Artifacts' - - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # - ${{ if eq(parameters.post_release, true) }}: - # # add release tags & push to github - # - task: GithubRelease@1 - # inputs: - # githubConnection: 'blobfuse-git-rel' - # repositoryName: 'Azure/azure-storage-fuse' - # action: 'edit' - - # target: '$(Build.SourceVersion)' - # tagSource: 'userSpecifiedTag' - - # title: ${{ parameters.tag }} - # tag: ${{ parameters.tag }} - - # assets: | - # $(Build.ArtifactStagingDirectory)/blobfuse2/* - # changeLogCompareToRelease: 'lastFullRelease' - # changeLogType: 'commitBased' - # isDraft: ${{ parameters.draft }} - # isPreRelease: ${{ parameters.prerelease }} - # assetUploadMode: replace - - # - ${{ if eq(parameters.publish_artifacts, true) }}: - # - stage: PublishArtifacts - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: PublishArtifacts - # timeoutInMinutes: 120 - # pool: - # vmImage: 'ubuntu-22.04' - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - ${{ if eq(parameters.post_release, true) }}: + # add release tags & push to github + - task: GithubRelease@1 + inputs: + githubConnection: 'blobfuse-git-rel' + repositoryName: 'Azure/azure-storage-fuse' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azcopy/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZURE_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'BlobFuse/Blobfuse' - # displayName: 'Connect to PMC artifact' - - # - script: | - # pip install pmc-cli - # displayName: 'Install pmc-cli' - - # - task: DownloadSecureFile@1 - # name: pmcCertificate - # displayName: 'Download pmc pem file' - # inputs: - # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - # - task: DownloadSecureFile@1 - # name: settings - # displayName: 'Download settings.toml file' - # inputs: - # secureFile: 'settings.toml' - - # - script: | - # pmc --version - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - # if [ $? -ne 0 ]; then - # exit 1 - # fi - # displayName: 'Test PMC installation' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Signed Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # cd mariner - # for f in ./blobfuse2*fuse3*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done - # ls -lRt - # mv blobfuse2*.rpm .. - # cd .. - # rm -r mariner/ - # displayName: 'Rename Mariner binaries' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # displayName: 'List Artifacts' - - # - script: | - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload blobfuse2-signed - # displayName: 'Upload packages' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ - - # - script: | - # wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv - # cat ./packages.csv - # displayName: 'Fetch packages.csv' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ - - # - script: | - # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` - # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" - - # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` - # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" - - # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` - # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" - - # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` - # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" - - # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` - # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" - - # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` - # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" - - # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` - # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" - - # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` - # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Add uploaded packages to repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - # - script: | - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Publish the repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'BlobFuse/Blobfuse' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + ls -lRt + mv blobfuse2*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed + displayName: 'Upload packages' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` + echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + + fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` + echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" + + fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` + echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" + + fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` + echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" + + fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` + echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" + + fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` + echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" + + marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` + echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" + + marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` + echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" + + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + - script: | + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ # - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts From d0a06b7f3c288d126a0cc3ff99c2bad6382eba58 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 16:11:56 +0530 Subject: [PATCH 048/513] add build job for arm64 --- release-pipelines.yml | 368 +++++++++++++++++++++--------------------- 1 file changed, 184 insertions(+), 184 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0a6d4c509..5205cac35 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -779,198 +779,198 @@ stages: # # TestArtifacts ends here - - stage: ReleaseArtifacts - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseBlobfuse - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - stage: ReleaseArtifacts + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') + # jobs: + # - job: ReleaseBlobfuse + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azcopy' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azcopy' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + # displayName: 'List Artifacts' - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - - ${{ if eq(parameters.post_release, true) }}: - # add release tags & push to github - - task: GithubRelease@1 - inputs: - githubConnection: 'blobfuse-git-rel' - repositoryName: 'Azure/azure-storage-fuse' - action: 'edit' - - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' - - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} - - assets: | - $(Build.ArtifactStagingDirectory)/azcopy/* - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace - - - ${{ if eq(parameters.publish_artifacts, true) }}: - - stage: PublishArtifacts - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: PublishArtifacts - timeoutInMinutes: 120 - pool: - vmImage: 'ubuntu-22.04' - variables: - - group: AZURE_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - ${{ if eq(parameters.post_release, true) }}: + # # add release tags & push to github + # - task: GithubRelease@1 + # inputs: + # githubConnection: 'blobfuse-git-rel' + # repositoryName: 'Azure/azure-storage-fuse' + # action: 'edit' + + # target: '$(Build.SourceVersion)' + # tagSource: 'userSpecifiedTag' + + # title: ${{ parameters.tag }} + # tag: ${{ parameters.tag }} + + # assets: | + # $(Build.ArtifactStagingDirectory)/azcopy/* + # changeLogCompareToRelease: 'lastFullRelease' + # changeLogType: 'commitBased' + # isDraft: ${{ parameters.draft }} + # isPreRelease: ${{ parameters.prerelease }} + # assetUploadMode: replace + + # - ${{ if eq(parameters.publish_artifacts, true) }}: + # - stage: PublishArtifacts + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: PublishArtifacts + # timeoutInMinutes: 120 + # pool: + # vmImage: 'ubuntu-22.04' + # variables: + # - group: AZURE_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - checkout: none - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'BlobFuse/Blobfuse' - displayName: 'Connect to PMC artifact' - - - script: | - pip install pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Signed Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - cd mariner - for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done - ls -lRt - mv blobfuse2*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - displayName: 'List Artifacts' - - - script: | - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed - displayName: 'Upload packages' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv - cat ./packages.csv - displayName: 'Fetch packages.csv' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` - echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" - - fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` - echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" - - fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` - echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" - - fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` - echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" - - fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` - echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" - - fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` - echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" - - marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` - echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" - - marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` - echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv) - displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - - script: | - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv) - displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + # steps: + # - checkout: none + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'BlobFuse/Blobfuse' + # displayName: 'Connect to PMC artifact' + + # - script: | + # pip install pmc-cli + # displayName: 'Install pmc-cli' + + # - task: DownloadSecureFile@1 + # name: pmcCertificate + # displayName: 'Download pmc pem file' + # inputs: + # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + # - task: DownloadSecureFile@1 + # name: settings + # displayName: 'Download settings.toml file' + # inputs: + # secureFile: 'settings.toml' + + # - script: | + # pmc --version + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + # if [ $? -ne 0 ]; then + # exit 1 + # fi + # displayName: 'Test PMC installation' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Signed Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # cd mariner + # for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + # ls -lRt + # mv blobfuse2*.rpm .. + # cd .. + # rm -r mariner/ + # displayName: 'Rename Mariner binaries' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # displayName: 'List Artifacts' + + # - script: | + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed + # displayName: 'Upload packages' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv + # cat ./packages.csv + # displayName: 'Fetch packages.csv' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` + # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + + # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` + # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" + + # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` + # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" + + # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` + # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" + + # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` + # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" + + # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` + # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" + + # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` + # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" + + # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` + # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" + + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Add uploaded packages to repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - script: | + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ # - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts From cce4bf8122525fb6f89dcdc9e3530f869427c15a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 16:48:12 +0530 Subject: [PATCH 049/513] add ut and e2etest stage --- release-pipelines.yml | 343 ++++++++++++++++++++++++++++++++---------- 1 file changed, 263 insertions(+), 80 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5205cac35..995fc0f9f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -43,9 +43,257 @@ trigger: none pr: none stages: + - stage: RunUT + - job: Test_On_Ubuntu + variables: + isMutexSet: 'false' + # allow maximum build time, in case we have build congestion + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + name: 'Set_up_Python' + inputs: + versionSpec: '3.7' + - task: GoTool@0 + name: 'Set_up_Golang' + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - task: DownloadSecureFile@1 + name: ciGCSServiceAccountKey + displayName: 'Download GCS Service Account Key' + inputs: + secureFile: 'ci-gcs-dev.json' + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Install dependencies' + - script: | + pip install azure-storage-blob==12.12.0 + # set the variable to indicate that the mutex is being acquired + # note: we set it before acquiring the mutex to ensure we release the mutex. + # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # mutex but before we set this variable. + # setting this before will always work since it is valid to break an un-acquired mutex. + echo '##vso[task.setvariable variable=isMutexSet]true' + # acquire the mutex before running live tests to avoid conflicts + python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + name: 'Acquire_the_distributed_mutex' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'cmd' + coverage_name: 'cmd' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common' + coverage_name: 'common' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common/parallel' + coverage_name: 'parallel' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'ste' + coverage_name: 'ste' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'sddl' + coverage_name: 'sddl' + - script: | + GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + go build -tags "netgo" -o test-validator ./testSuite/ + mkdir test-temp + mkdir coverage + export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + keyctl session test python ./testSuite/scripts/run.py + + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o smoke_coverage.txt + echo 'Formatting coverage to json format' + $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + echo 'Formatting coverage to xml format' + $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + name: 'Run_smoke_tests' + env: + ACCOUNT_NAME: $(ACCOUNT_NAME) + ACCOUNT_KEY: $(ACCOUNT_KEY) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + FILESYSTEM_URL: $(FILESYSTEM_URL) + OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + SHARE_SAS_URL: $(SHARE_SAS_URL) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + condition: succeededOrFailed() + # Smoke Tests Publishing + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + - script: | + pip install azure-storage-blob==12.12.0 + python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + name: 'Release_the_distributed_mutex' + # this runs even if the job was canceled (only if the mutex was acquired by this job) + condition: and(always(), eq(variables['isMutexSet'], 'true')) + + - stage: E2ETest + - job: E2E_Test + timeoutInMinutes: 360 + # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + build_name: 'azcopy_linux_amd64' + display_name: "Linux" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + Windows: + imageName: 'windows-2019' + build_name: 'azcopy_windows_amd64.exe' + display_name: "Windows" + type: 'windows' + go_path: 'C:\Users\VssAdministrator\go\bin\' + suffix: '.exe' + run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + output_test_logs: 'cat test.txt' + save_exit_code: 'set exit_code=%errorlevel%' + return_exit_code: 'exit %exit_code%' + go_build: 'go build -cover -o $(build_name)' + MacOS: + imageName: 'macos-11' + build_name: 'azcopy_darwin_amd64' + display_name: "MacOS" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + pool: + vmImage: $(imageName) + + steps: + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + pwsh: 'true' + displayName: 'Install Powershell Az Module' + - task: GoTool@0 + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Installing dependencies' + - bash: | + echo "##vso[task.setvariable variable=CGO_ENABLED]0" + displayName: 'Set CGO_ENABLED for Windows' + condition: eq(variables.type, 'windows') + - bash: | + npm install -g azurite + mkdir azurite + azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + # Running E2E Tests on AMD64 + - script: | + mkdir coverage + echo 'Building executable' + $(go_build) + echo 'Running tests' + $(run_e2e) + $(save_exit_code) + echo "Generating junit report" + cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + $(output_test_logs) + echo 'Formatting coverage to json format' + $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + echo 'Formatting coverage to xml format' + $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + $(return_exit_code) + env: + AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + displayName: 'E2E Test $(display_name) - AMD64' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish logs' + condition: succeededOrFailed() + inputs: + pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + artifactName: logs + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + testRunTitle: 'Go on $(display_name)' + + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + - stage: BuildArtifacts jobs: - - job: Set_1 + - job: Ubuntu timeoutInMinutes: 120 strategy: matrix: @@ -92,10 +340,17 @@ stages: GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + - script: | cp NOTICE.txt $(work_dir) displayName: 'Copy NOTICE.txt' - condition: eq(variables.type, 'linux') # place the generated binary files & any additional files in appropriate locations - script: | @@ -121,7 +376,7 @@ stages: mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) - displayName: 'Make deb Package' + displayName: 'Make AMD64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ @@ -132,84 +387,12 @@ stages: mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) - displayName: 'Make rpm Package' + displayName: 'Make AMD64 rpm Package' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Linux_arm64: - vmImage: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' - - - script: | - cp NOTICE.txt $(work_dir) - displayName: 'Copy NOTICE.txt' - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - cp -r $(work_dir) pkgDir/usr/bin/ - displayName: 'Accumulate pkg files' - - # TODO: remove after testing - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ @@ -219,7 +402,7 @@ stages: mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) - displayName: 'Make deb Package' + displayName: 'Make ARM64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ @@ -230,7 +413,7 @@ stages: mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) - displayName: 'Make rpm Package' + displayName: 'Make ARM64 rpm Package' - task: PublishBuildArtifacts@1 inputs: @@ -243,7 +426,7 @@ stages: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' - + # BuildArtifacts end here - stage: SignArtifacts From 68827df1d8a771cad6b42a55c1300a278911ca33 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 17:08:07 +0530 Subject: [PATCH 050/513] add ut and e2etest stage --- release-pipelines.yml | 459 +++++++++++++++++++++--------------------- 1 file changed, 234 insertions(+), 225 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 995fc0f9f..a9a781c28 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -43,232 +43,241 @@ trigger: none pr: none stages: - - stage: RunUT - - job: Test_On_Ubuntu - variables: - isMutexSet: 'false' - # allow maximum build time, in case we have build congestion - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - name: 'Set_up_Python' - inputs: - versionSpec: '3.7' - - task: GoTool@0 - name: 'Set_up_Golang' - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - task: DownloadSecureFile@1 - name: ciGCSServiceAccountKey - displayName: 'Download GCS Service Account Key' - inputs: - secureFile: 'ci-gcs-dev.json' - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Install dependencies' - - script: | - pip install azure-storage-blob==12.12.0 - # set the variable to indicate that the mutex is being acquired - # note: we set it before acquiring the mutex to ensure we release the mutex. - # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # mutex but before we set this variable. - # setting this before will always work since it is valid to break an un-acquired mutex. - echo '##vso[task.setvariable variable=isMutexSet]true' - # acquire the mutex before running live tests to avoid conflicts - python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - name: 'Acquire_the_distributed_mutex' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'cmd' - coverage_name: 'cmd' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common' - coverage_name: 'common' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common/parallel' - coverage_name: 'parallel' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'ste' - coverage_name: 'ste' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'sddl' - coverage_name: 'sddl' - - script: | - GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - go build -tags "netgo" -o test-validator ./testSuite/ - mkdir test-temp - mkdir coverage - export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - keyctl session test python ./testSuite/scripts/run.py + - stage: Run Tests + jobs: + - job: UT_On_Ubuntu + variables: + isMutexSet: 'false' + # allow maximum build time, in case we have build congestion + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + + steps: + - task: UsePythonVersion@0 + name: 'Set_up_Python' + inputs: + versionSpec: '3.7' + + - task: GoTool@0 + name: 'Set_up_Golang' + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + - task: DownloadSecureFile@1 + name: ciGCSServiceAccountKey + displayName: 'Download GCS Service Account Key' + inputs: + secureFile: 'ci-gcs-dev.json' + + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Install dependencies' + + - script: | + pip install azure-storage-blob==12.12.0 + # set the variable to indicate that the mutex is being acquired + # note: we set it before acquiring the mutex to ensure we release the mutex. + # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # mutex but before we set this variable. + # setting this before will always work since it is valid to break an un-acquired mutex. + echo '##vso[task.setvariable variable=isMutexSet]true' + # acquire the mutex before running live tests to avoid conflicts + python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + name: 'Acquire_the_distributed_mutex' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'cmd' + coverage_name: 'cmd' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common' + coverage_name: 'common' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common/parallel' + coverage_name: 'parallel' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'ste' + coverage_name: 'ste' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'sddl' + coverage_name: 'sddl' + + - script: | + GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + go build -tags "netgo" -o test-validator ./testSuite/ + mkdir test-temp + mkdir coverage + export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + keyctl session test python ./testSuite/scripts/run.py - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o smoke_coverage.txt - echo 'Formatting coverage to json format' - $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - echo 'Formatting coverage to xml format' - $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - name: 'Run_smoke_tests' - env: - ACCOUNT_NAME: $(ACCOUNT_NAME) - ACCOUNT_KEY: $(ACCOUNT_KEY) - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - FILESYSTEM_URL: $(FILESYSTEM_URL) - OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - SHARE_SAS_URL: $(SHARE_SAS_URL) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - condition: succeededOrFailed() - # Smoke Tests Publishing - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - - script: | - pip install azure-storage-blob==12.12.0 - python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - name: 'Release_the_distributed_mutex' - # this runs even if the job was canceled (only if the mutex was acquired by this job) - condition: and(always(), eq(variables['isMutexSet'], 'true')) - - - stage: E2ETest - - job: E2E_Test - timeoutInMinutes: 360 - # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - strategy: - matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - build_name: 'azcopy_linux_amd64' - display_name: "Linux" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - Windows: - imageName: 'windows-2019' - build_name: 'azcopy_windows_amd64.exe' - display_name: "Windows" - type: 'windows' - go_path: 'C:\Users\VssAdministrator\go\bin\' - suffix: '.exe' - run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - output_test_logs: 'cat test.txt' - save_exit_code: 'set exit_code=%errorlevel%' - return_exit_code: 'exit %exit_code%' - go_build: 'go build -cover -o $(build_name)' - MacOS: - imageName: 'macos-11' - build_name: 'azcopy_darwin_amd64' - display_name: "MacOS" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - pool: - vmImage: $(imageName) - - steps: - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - pwsh: 'true' - displayName: 'Install Powershell Az Module' - - task: GoTool@0 - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Installing dependencies' - - bash: | - echo "##vso[task.setvariable variable=CGO_ENABLED]0" - displayName: 'Set CGO_ENABLED for Windows' - condition: eq(variables.type, 'windows') - - bash: | - npm install -g azurite - mkdir azurite - azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' - # Running E2E Tests on AMD64 - - script: | - mkdir coverage - echo 'Building executable' - $(go_build) - echo 'Running tests' - $(run_e2e) - $(save_exit_code) - echo "Generating junit report" - cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - $(output_test_logs) - echo 'Formatting coverage to json format' - $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - echo 'Formatting coverage to xml format' - $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - $(return_exit_code) - env: - AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - displayName: 'E2E Test $(display_name) - AMD64' + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o smoke_coverage.txt + echo 'Formatting coverage to json format' + $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + echo 'Formatting coverage to xml format' + $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + name: 'Run_smoke_tests' + env: + ACCOUNT_NAME: $(ACCOUNT_NAME) + ACCOUNT_KEY: $(ACCOUNT_KEY) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + FILESYSTEM_URL: $(FILESYSTEM_URL) + OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + SHARE_SAS_URL: $(SHARE_SAS_URL) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + condition: succeededOrFailed() + # Smoke Tests Publishing + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + - script: | + pip install azure-storage-blob==12.12.0 + python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + name: 'Release_the_distributed_mutex' + # this runs even if the job was canceled (only if the mutex was acquired by this job) + condition: and(always(), eq(variables['isMutexSet'], 'true')) + - job: E2E_Test + timeoutInMinutes: 360 + # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + build_name: 'azcopy_linux_amd64' + display_name: "Linux" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + Windows: + imageName: 'windows-2019' + build_name: 'azcopy_windows_amd64.exe' + display_name: "Windows" + type: 'windows' + go_path: 'C:\Users\VssAdministrator\go\bin\' + suffix: '.exe' + run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + output_test_logs: 'cat test.txt' + save_exit_code: 'set exit_code=%errorlevel%' + return_exit_code: 'exit %exit_code%' + go_build: 'go build -cover -o $(build_name)' + MacOS: + imageName: 'macos-11' + build_name: 'azcopy_darwin_amd64' + display_name: "MacOS" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + pool: + vmImage: $(imageName) + + steps: + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + pwsh: 'true' + displayName: 'Install Powershell Az Module' + - task: GoTool@0 + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Installing dependencies' + - bash: | + echo "##vso[task.setvariable variable=CGO_ENABLED]0" + displayName: 'Set CGO_ENABLED for Windows' + condition: eq(variables.type, 'windows') + - bash: | + npm install -g azurite + mkdir azurite + azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + # Running E2E Tests on AMD64 + - script: | + mkdir coverage + echo 'Building executable' + $(go_build) + echo 'Running tests' + $(run_e2e) + $(save_exit_code) + echo "Generating junit report" + cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + $(output_test_logs) + echo 'Formatting coverage to json format' + $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + echo 'Formatting coverage to xml format' + $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + $(return_exit_code) + env: + AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + displayName: 'E2E Test $(display_name) - AMD64' - task: PublishBuildArtifacts@1 displayName: 'Publish logs' From 2cb7c33a7ef1424a4c59df86e2b13b621f85efd6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 17:09:35 +0530 Subject: [PATCH 051/513] add ut and e2etest stage --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a9a781c28..1d5011b6a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -354,8 +354,8 @@ stages: displayName: 'Generate Linux AMD64 SE Integration' - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - script: | cp NOTICE.txt $(work_dir) From 749f9e277fad59a30900ce9068e70b5982e573bd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 17:13:16 +0530 Subject: [PATCH 052/513] add ut and e2etest stage --- release-pipelines.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1d5011b6a..f59c77cc2 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -279,26 +279,26 @@ stages: NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) displayName: 'E2E Test $(display_name) - AMD64' - - task: PublishBuildArtifacts@1 - displayName: 'Publish logs' - condition: succeededOrFailed() - inputs: - pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - artifactName: logs - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testRunner: JUnit - testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - testRunTitle: 'Go on $(display_name)' - - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + - task: PublishBuildArtifacts@1 + displayName: 'Publish logs' + condition: succeededOrFailed() + inputs: + pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + artifactName: logs + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + testRunTitle: 'Go on $(display_name)' + + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts jobs: From 9d8e86743359b52c338387e7eb73d6a77f74aed9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 17:13:58 +0530 Subject: [PATCH 053/513] add ut and e2etest stage --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f59c77cc2..2f89c06e0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -43,7 +43,7 @@ trigger: none pr: none stages: - - stage: Run Tests + - stage: Run_Tests jobs: - job: UT_On_Ubuntu variables: From 532a68646094fc87656c05a164fd096bdfff2640 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 17:37:22 +0530 Subject: [PATCH 054/513] comment the uts stage --- release-pipelines.yml | 490 +++++++++++++++++++++--------------------- 1 file changed, 245 insertions(+), 245 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9ad4f40cd..46cc36a97 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -43,262 +43,262 @@ trigger: none pr: none stages: - - stage: RunTests - jobs: - - job: UT_On_Ubuntu - variables: - isMutexSet: 'false' - # allow maximum build time, in case we have build congestion - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' + # - stage: RunTests + # jobs: + # - job: UT_On_Ubuntu + # variables: + # isMutexSet: 'false' + # # allow maximum build time, in case we have build congestion + # timeoutInMinutes: 360 + # pool: + # vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - name: 'Set_up_Python' - inputs: - versionSpec: '3.7' + # steps: + # - task: UsePythonVersion@0 + # name: 'Set_up_Python' + # inputs: + # versionSpec: '3.7' - - task: GoTool@0 - name: 'Set_up_Golang' - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - task: GoTool@0 + # name: 'Set_up_Golang' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - task: DownloadSecureFile@1 - name: ciGCSServiceAccountKey - displayName: 'Download GCS Service Account Key' - inputs: - secureFile: 'ci-gcs-dev.json' + # - task: DownloadSecureFile@1 + # name: ciGCSServiceAccountKey + # displayName: 'Download GCS Service Account Key' + # inputs: + # secureFile: 'ci-gcs-dev.json' - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Install dependencies' + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Install dependencies' - - script: | - pip install azure-storage-blob==12.12.0 - # set the variable to indicate that the mutex is being acquired - # note: we set it before acquiring the mutex to ensure we release the mutex. - # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # mutex but before we set this variable. - # setting this before will always work since it is valid to break an un-acquired mutex. - echo '##vso[task.setvariable variable=isMutexSet]true' - # acquire the mutex before running live tests to avoid conflicts - python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - name: 'Acquire_the_distributed_mutex' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'cmd' - coverage_name: 'cmd' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common' - coverage_name: 'common' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common/parallel' - coverage_name: 'parallel' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'ste' - coverage_name: 'ste' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'sddl' - coverage_name: 'sddl' + # - script: | + # pip install azure-storage-blob==12.12.0 + # # set the variable to indicate that the mutex is being acquired + # # note: we set it before acquiring the mutex to ensure we release the mutex. + # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # # mutex but before we set this variable. + # # setting this before will always work since it is valid to break an un-acquired mutex. + # echo '##vso[task.setvariable variable=isMutexSet]true' + # # acquire the mutex before running live tests to avoid conflicts + # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + # name: 'Acquire_the_distributed_mutex' + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'cmd' + # coverage_name: 'cmd' - - script: | - GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - go build -tags "netgo" -o test-validator ./testSuite/ - mkdir test-temp - mkdir coverage - export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - keyctl session test python ./testSuite/scripts/run.py + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common' + # coverage_name: 'common' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common/parallel' + # coverage_name: 'parallel' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'ste' + # coverage_name: 'ste' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'sddl' + # coverage_name: 'sddl' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + # go build -tags "netgo" -o test-validator ./testSuite/ + # mkdir test-temp + # mkdir coverage + # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + # export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + # keyctl session test python ./testSuite/scripts/run.py - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o smoke_coverage.txt - echo 'Formatting coverage to json format' - $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - echo 'Formatting coverage to xml format' - $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - name: 'Run_smoke_tests' - env: - ACCOUNT_NAME: $(ACCOUNT_NAME) - ACCOUNT_KEY: $(ACCOUNT_KEY) - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - FILESYSTEM_URL: $(FILESYSTEM_URL) - OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - SHARE_SAS_URL: $(SHARE_SAS_URL) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - condition: succeededOrFailed() - # Smoke Tests Publishing - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - - script: | - pip install azure-storage-blob==12.12.0 - python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - name: 'Release_the_distributed_mutex' - # this runs even if the job was canceled (only if the mutex was acquired by this job) - condition: and(always(), eq(variables['isMutexSet'], 'true')) - - job: E2E_Test - timeoutInMinutes: 360 - # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - strategy: - matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - build_name: 'azcopy_linux_amd64' - display_name: "Linux" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - Windows: - imageName: 'windows-2019' - build_name: 'azcopy_windows_amd64.exe' - display_name: "Windows" - type: 'windows' - go_path: 'C:\Users\VssAdministrator\go\bin\' - suffix: '.exe' - run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - output_test_logs: 'cat test.txt' - save_exit_code: 'set exit_code=%errorlevel%' - return_exit_code: 'exit %exit_code%' - go_build: 'go build -cover -o $(build_name)' - MacOS: - imageName: 'macos-11' - build_name: 'azcopy_darwin_amd64' - display_name: "MacOS" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - pool: - vmImage: $(imageName) + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o smoke_coverage.txt + # echo 'Formatting coverage to json format' + # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + # echo 'Formatting coverage to xml format' + # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + # name: 'Run_smoke_tests' + # env: + # ACCOUNT_NAME: $(ACCOUNT_NAME) + # ACCOUNT_KEY: $(ACCOUNT_KEY) + # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + # FILESYSTEM_URL: $(FILESYSTEM_URL) + # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + # SHARE_SAS_URL: $(SHARE_SAS_URL) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # condition: succeededOrFailed() + # # Smoke Tests Publishing + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + # - script: | + # pip install azure-storage-blob==12.12.0 + # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + # name: 'Release_the_distributed_mutex' + # # this runs even if the job was canceled (only if the mutex was acquired by this job) + # condition: and(always(), eq(variables['isMutexSet'], 'true')) + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) - steps: - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - pwsh: 'true' - displayName: 'Install Powershell Az Module' - - task: GoTool@0 - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Installing dependencies' - - bash: | - echo "##vso[task.setvariable variable=CGO_ENABLED]0" - displayName: 'Set CGO_ENABLED for Windows' - condition: eq(variables.type, 'windows') - - bash: | - npm install -g azurite - mkdir azurite - azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' - # Running E2E Tests on AMD64 - - script: | - mkdir coverage - echo 'Building executable' - $(go_build) - echo 'Running tests' - $(run_e2e) - $(save_exit_code) - echo "Generating junit report" - cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - $(output_test_logs) - echo 'Formatting coverage to json format' - $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - echo 'Formatting coverage to xml format' - $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - $(return_exit_code) - env: - AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - displayName: 'E2E Test $(display_name) - AMD64' + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' - - task: PublishBuildArtifacts@1 - displayName: 'Publish logs' - condition: succeededOrFailed() - inputs: - pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - artifactName: logs + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testRunner: JUnit - testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - testRunTitle: 'Go on $(display_name)' + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts dependsOn: RunTests From 22616a923d42566e88e958c889c65cb73a531588 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 17:38:04 +0530 Subject: [PATCH 055/513] comment the uts stage --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 46cc36a97..04e2d5ad6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -301,8 +301,8 @@ stages: # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts - dependsOn: RunTests - condition: succeeded('RunTests') + #dependsOn: RunTests + #condition: succeeded('RunTests') jobs: - job: Ubuntu timeoutInMinutes: 120 From a0b2dba25f4e40e989e6d75031e0c6edc04f352d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 18:24:48 +0530 Subject: [PATCH 056/513] comment the uts stage --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 04e2d5ad6..fe1b5fb0f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -342,7 +342,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems build-essential rpm -y + sudo apt-get install ruby ruby-dev rubygems pkg-config build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" From cb5da055b08a1f30ce4cde7641144ee404cfe257 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 21 May 2024 18:35:15 +0530 Subject: [PATCH 057/513] comment the uts stage --- release-pipelines.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index fe1b5fb0f..ca94ee085 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -347,6 +347,11 @@ stages: sudo gem install fpm -V displayName: "Installing Dependencies" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' From a2ad58146d610b0a737bc10992c1d4a0e7329b7c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 15:46:26 +0530 Subject: [PATCH 058/513] comment the uts stage --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ca94ee085..ed1f10218 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -357,11 +357,11 @@ stages: displayName: 'Generate Linux AMD64' - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" displayName: 'Generate Linux AMD64 SE Integration' - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64" + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' - script: | From 902d92c019f99b61f8fb5517f826172b36914f1a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 15:54:41 +0530 Subject: [PATCH 059/513] comment the uts stage --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ed1f10218..abdfc6538 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -356,9 +356,9 @@ stages: GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" From 646129f096b38ed6172526a30c0e63bd9d6d50db Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 16:50:38 +0530 Subject: [PATCH 060/513] comment the uts stage --- release-pipelines.yml | 493 +++++++++++++++--------------------------- 1 file changed, 179 insertions(+), 314 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index abdfc6538..0531d4349 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -345,6 +345,7 @@ stages: sudo apt-get install ruby ruby-dev rubygems pkg-config build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V + sudo apt-get install libglib2.0-dev displayName: "Installing Dependencies" # get glibc version with which build is done @@ -356,9 +357,9 @@ stages: GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" @@ -384,7 +385,7 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ + fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDir/ \ -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -395,7 +396,7 @@ stages: displayName: 'Make AMD64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDir/ \ -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -410,7 +411,7 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ + fpm -s dir -t deb -n azcopy_linux_arm64 -C pkgDir/ \ -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -421,7 +422,7 @@ stages: displayName: 'Make ARM64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + fpm -s dir -t rpm -n azcopy_linux_arm64 --rpm-digest sha256 -C pkgDir/ \ -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -626,31 +627,24 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy*.deb - sudo dpkg -i azcopy*.deb + sudo dpkg --info azcopy_linux_arm64*.deb + sudo dpkg -i azcopy_linux_arm64*.deb sudo apt-get install build-essential -y - displayName: 'Install Package' + displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - script: | - azcopy --version + azcopy_linux_arm64 --version displayName: 'Check Version' - script: | - azcopy --help + azcopy_linux_arm64 --help displayName: 'Check Help' # # publishing the artifacts generated @@ -659,324 +653,195 @@ stages: # artifactName: 'blobfuse2' # displayName: 'Publish Artifacts' - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-18: - # vmImage: 'Ubuntu-18.04' - # fuselib: 'libfuse-dev' - # fuse-version: 'fuse' - # tags: 'fuse2' - # container: 'test-cnt-ubn-18' - # AgentName: "blobfuse-ubuntu18" - # Ubuntu-20: - # vmImage: 'Ubuntu-20.04' - # fuse-version: 'fuse3' - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - - # steps: - # - checkout: none - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*x86_64.deb - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # if [ $(AgentName) == "blobfuse-ubuntu18" ]; then - # echo "Generating for Debian 9/10" - # f=`ls ./blobfuse2*$(vmImage)*.deb` - # cp "$f" $(sed 's:Ubuntu-18.04:Debian-9.0:' <<< "$f") - # cp "$f" $(sed 's:Ubuntu-18.04:Debian-10.0:' <<< "$f") - # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - # ls -l $(Build.ArtifactStagingDirectory) - # rm -rf ./blobfuse2*Debian-*.deb - # else - # echo "Generating for Debian 11" - # f=`ls ./blobfuse2*$(vmImage)*.deb` - # cp "$f" $(sed 's:Ubuntu-20.04:Debian-11.0:' <<< "$f") - # cp ./blobfuse2*Debian-*.deb $(Build.ArtifactStagingDirectory) - # ls -l $(Build.ArtifactStagingDirectory) - # rm -rf ./blobfuse2*Debian-*.deb - # fi - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed - - # - script: | - # sudo dpkg --info blobfuse2*.deb - # sudo apt-get install $(fuse-version) build-essential -y - # sudo dpkg -i blobfuse2*.deb - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' - - # - job: Set_2_ARM - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22-ARM64: - # vmImage: 'Ubuntu-22.04' - # fuse-version: 'fuse3' - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) + + - job: Set_2_ARM + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22-ARM64: + vmImage: 'Ubuntu-22.04' + fuse-version: 'fuse3' + fuselib: 'libfuse3-dev' + tags: 'fuse3' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + variables: + - group: NightlyBlobFuse + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/fusetmp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/fusetmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-fuse - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/blobfuse2*$(tags)*arm64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'blobfuse2-signed' + downloadPath: $(root_dir) + itemPattern: blobfuse2-signed/blobfuse2*$(tags)*arm64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/blobfuse2-signed + - script: | + for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/blobfuse2-signed - # - script: | - # sudo dpkg --info blobfuse2*.deb - # sudo apt-get install $(fuse-version) build-essential -y - # sudo dpkg -i blobfuse2*.deb - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo dpkg --info blobfuse2*.deb + sudo apt-get install $(fuse-version) build-essential -y + sudo dpkg -i blobfuse2*.deb + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) + - template: 'azure-pipeline-templates/release-distro-tests.yml' + parameters: + root_dir: $(root_dir) + work_dir: $(work_dir) + mount_dir: $(mount_dir) + temp_dir: $(temp_dir) + container: $(container) - # # Create container for blobfuse and publish it - # - script: | - # sudo apt-get install $(tags) $(fuselib) -y - # chmod 777 *.sh - # ./dockerinstall.sh - # ./buildcontainer.sh DockerfileARM arm64 - # ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) arm64 - # displayName: "Create and publish ARM container" - # workingDirectory: $(work_dir)/docker - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' - - # - job: Set_8 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # fuselib: 'libfuse3-dev' - # fuse-version: 'fuse3' - # tags: 'fuse3' - # container: "test-cnt-mari-2" + # Create container for blobfuse and publish it + - script: | + sudo apt-get install $(tags) $(fuselib) -y + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh DockerfileARM arm64 + ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) arm64 + displayName: "Create and publish ARM container" + workingDirectory: $(work_dir)/docker + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'blobfuse2' + displayName: 'Publish Artifacts' - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + tags: 'tags' + container: "test-cnt-mari-2" - # variables: - # - group: NightlyBlobFuse - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) - # steps: - # - checkout: none + variables: + - group: AZURE_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/azcopytmp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-fuse + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-fuse - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/mariner/blobfuse2*$(tags)*x86_64.rpm + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/blobfuse2-signed/mariner + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/mariner/azcopy*$(tags)*x86_64.rpm - # - script: | - # sudo rpm -qip blobfuse2*.rpm - # sudo tdnf install gcc build-essential fuse3 fuse3-devel -y - # sudo rpm -i blobfuse2*.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/blobfuse2-signed/mariner + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed/mariner - # - template: 'azure-pipeline-templates/release-distro-tests.yml' - # parameters: - # root_dir: $(root_dir) - # work_dir: $(work_dir) - # mount_dir: $(mount_dir) - # temp_dir: $(temp_dir) - # container: $(container) + - script: | + sudo rpm -qip blobfuse2*.rpm + sudo tdnf install gcc build-essential fuse3 fuse3-devel -y + sudo rpm -i azcopy*.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-signed/mariner - # # TestArtifacts ends here + # TestArtifacts ends here # - stage: ReleaseArtifacts # dependsOn: TestArtifacts From 791e9a6429dc575c22b1603375d15621d94f13b6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 16:56:51 +0530 Subject: [PATCH 061/513] comment the uts stage --- go_installer.sh | 15 ++++++++++ release-pipelines.yml | 65 ++++++++++++++----------------------------- 2 files changed, 36 insertions(+), 44 deletions(-) create mode 100644 go_installer.sh diff --git a/go_installer.sh b/go_installer.sh new file mode 100644 index 000000000..80384690a --- /dev/null +++ b/go_installer.sh @@ -0,0 +1,15 @@ +#!/bin/bash +work_dir=$(echo $1 | sed 's:/*$::') +version="1.22.1" +arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` + +if [ $arch != "arm64" ] +then + arch="amd64" +fi + +echo "Installing on : " $arch " Version : " $version +wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir" +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz +sudo ln -sf /usr/local/go/bin/go /usr/bin/go \ No newline at end of file diff --git a/release-pipelines.yml b/release-pipelines.yml index 0531d4349..e191531dc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -660,9 +660,6 @@ stages: matrix: Ubuntu-22-ARM64: vmImage: 'Ubuntu-22.04' - fuse-version: 'fuse3' - fuselib: 'libfuse3-dev' - tags: 'fuse3' container: 'test-cnt-ubn-22-arm64' AgentName: "blobfuse-ubn22-arm64" pool: @@ -671,28 +668,28 @@ stages: - ImageOverride -equals $(AgentName) variables: - - group: NightlyBlobFuse + - group: AZCOPY_ESRP_SECRET - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/fusetmp' + value: '$(System.DefaultWorkingDirectory)/azcopytmp' - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/fusetmpcache' + value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' steps: - checkout: none - script: | - git clone https://github.com/Azure/azure-storage-fuse + git clone https://github.com/Azure/azure-storage-azcopy displayName: 'Checkout Code' workingDirectory: $(root_dir) - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse + workingDirectory: $(root_dir)/azure-storage-azcopy # Custom script to install Go-lang - task: ShellScript@2 @@ -711,9 +708,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'blobfuse2-signed' + artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/blobfuse2*$(tags)*arm64.deb + itemPattern: azCopy-signed/azcopy_linux_arm64*$(tags)*arm64.deb - script: | ls -l @@ -722,44 +719,25 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/blobfuse2-signed + workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./blobfuse2*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./blobfuse2*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' - workingDirectory: $(root_dir)/blobfuse2-signed + workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info blobfuse2*.deb - sudo apt-get install $(fuse-version) build-essential -y - sudo dpkg -i blobfuse2*.deb + sudo dpkg --info azcopy_linux_arm64*.deb + sudo dpkg -i azcopy_linux_arm64*.deb displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) - - template: 'azure-pipeline-templates/release-distro-tests.yml' - parameters: - root_dir: $(root_dir) - work_dir: $(work_dir) - mount_dir: $(mount_dir) - temp_dir: $(temp_dir) - container: $(container) - - # Create container for blobfuse and publish it - - script: | - sudo apt-get install $(tags) $(fuselib) -y - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh DockerfileARM arm64 - ./publishcontainer.sh $(AZTEST_DOCKER_REG_USER) $(AZTEST_DOCKER_REG_PWD) arm64 - displayName: "Create and publish ARM container" - workingDirectory: $(work_dir)/docker - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' - job: Set_3 timeoutInMinutes: 120 @@ -804,7 +782,7 @@ stages: - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-fuse + workingDirectory: $(root_dir)/azure-storage-azcopy # Custom script to install Go-lang - task: ShellScript@2 @@ -835,8 +813,7 @@ stages: workingDirectory: $(root_dir)/azCopy-signed/mariner - script: | - sudo rpm -qip blobfuse2*.rpm - sudo tdnf install gcc build-essential fuse3 fuse3-devel -y + sudo rpm -qip azcopy*.rpm sudo rpm -i azcopy*.rpm displayName: 'Install Package' workingDirectory: $(root_dir)/azCopy-signed/mariner From 0229e0ef13ff38daf67c68b9d15fbc384d4a4a4d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 16:58:44 +0530 Subject: [PATCH 062/513] comment the uts stage --- release-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e191531dc..c350b5dfd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -668,7 +668,6 @@ stages: - ImageOverride -equals $(AgentName) variables: - - group: AZCOPY_ESRP_SECRET - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir @@ -756,7 +755,6 @@ stages: - ImageOverride -equals $(AgentName) variables: - - group: AZURE_ESRP_SECRET - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir From 1f8e1aced058fbd46fd1c07bc7ee9e05bf7f0885 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 17:07:53 +0530 Subject: [PATCH 063/513] comment the uts stage --- release-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c350b5dfd..f09fa721a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -342,10 +342,9 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems pkg-config build-essential rpm -y + sudo apt-get install ruby ruby-dev rubygems pkg-config libglib2.0-dev build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V - sudo apt-get install libglib2.0-dev displayName: "Installing Dependencies" # get glibc version with which build is done From fa17ae190962ca651a5a7a5d3a9672c387ee1f1c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 17:21:16 +0530 Subject: [PATCH 064/513] comment the uts stage --- go_installer.sh | 2 +- release-pipelines.yml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go_installer.sh b/go_installer.sh index 80384690a..cc4b970c6 100644 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.22.1" +version="1.21.8" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ] diff --git a/release-pipelines.yml b/release-pipelines.yml index f09fa721a..b9501b61f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -327,11 +327,12 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' + # Custom script to install Go-lang + - task: ShellScript@2 inputs: - version: $(AZCOPY_GOLANG_VERSION) + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - script: | echo 'Running GO Vet' From 8ba8eee0153ebcece5114c7ac409102055d2ecab Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 22 May 2024 17:36:29 +0530 Subject: [PATCH 065/513] comment the uts stage --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b9501b61f..a5adccc61 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -328,11 +328,11 @@ stages: steps: # Custom script to install Go-lang - - task: ShellScript@2 + - task: GoTool@0 + env: + GO111MODULE: 'on' inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + version: $(AZCOPY_GOLANG_VERSION) - script: | echo 'Running GO Vet' From c57cb72d2b95d75359c7586f4521ecad062801be Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 10:57:19 +0530 Subject: [PATCH 066/513] installing required package --- release-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a5adccc61..530660669 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -343,7 +343,8 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install ruby ruby-dev rubygems pkg-config libglib2.0-dev build-essential rpm -y + sudo apt-get install pkg-config libsecret-1-0 -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" @@ -667,7 +668,7 @@ stages: demands: - ImageOverride -equals $(AgentName) - variables: + variables: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir From cbfe774727937d84b1776f2d039ada0bcb3cd32e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 11:09:45 +0530 Subject: [PATCH 067/513] installing required package --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 530660669..0150c8076 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -343,7 +343,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-0 -y + sudo apt-get install pkg-config libsecret-1 -y sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V From fa6173b65545cd2f87c6ebeb47eefbea334c72e0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 11:21:53 +0530 Subject: [PATCH 068/513] installing required package --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0150c8076..e1cbed6ad 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -343,7 +343,7 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1 -y + sudo apt-get install pkg-config libsecret-1-dev -y sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V From 4e4ca976e6663d683984d9f7e59c10cfecd09a7f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 12:33:11 +0530 Subject: [PATCH 069/513] build artifacts changes --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e1cbed6ad..b19445890 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -391,7 +391,7 @@ stages: --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' @@ -402,7 +402,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -417,7 +417,7 @@ stages: --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' @@ -428,7 +428,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' From c907fc7a140fb71d35cb369b3d1d2dcd7b0d5721 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 12:34:39 +0530 Subject: [PATCH 070/513] test artifacts changes --- release-pipelines.yml | 162 +++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b19445890..4bde8633f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -417,8 +417,8 @@ stages: --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy_linux_arm64*.deb ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy_linux_arm64*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' @@ -428,8 +428,8 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy_linux_arm64*.rpm ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy_linux_arm64*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' @@ -739,83 +739,83 @@ stages: # artifactName: 'blobfuse2' # displayName: 'Publish Artifacts' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - tags: 'tags' - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - - steps: - - checkout: none - - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install Git' - - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-signed/mariner/azcopy*$(tags)*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed/mariner - - - script: | - sudo rpm -qip azcopy*.rpm - sudo rpm -i azcopy*.rpm - displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-signed/mariner + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # tags: 'tags' + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/azcopytmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' + + # steps: + # - checkout: none + + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install Git' + + # - script: | + # git clone https://github.com/Azure/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-signed/mariner/azcopy*$(tags)*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-signed/mariner + + # - script: | + # sudo rpm -qip azcopy*.rpm + # sudo rpm -i azcopy*.rpm + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/azCopy-signed/mariner # TestArtifacts ends here From d5c577d1890d2bf5a4a787a430dde92a44a52d12 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 13:11:48 +0530 Subject: [PATCH 071/513] test artifacts changes --- release-pipelines.yml | 51 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4bde8633f..2efb3a83a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -382,30 +382,30 @@ stages: workingDirectory: $(root_dir) - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "AzCopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy_linux_amd64*.deb ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + # cp ./azcopy_linux_amd64*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy_linux_amd64*.rpm ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # cp ./azcopy_linux_amd64*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' # using fpm tool for packaging of our binary & performing post-install operations @@ -433,6 +433,11 @@ stages: workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' + - script: | + cp ./azcopy_linux_se_amd64*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add SE binary to signing stage' + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' From 028902e2a0084788c57f246731864f84cd2ee985 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 13:34:54 +0530 Subject: [PATCH 072/513] test artifacts changes --- release-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2efb3a83a..62474f3dc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -365,6 +365,10 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' + + - script: | + ./$(work_dir)/azcopy_linux_arm64 --version + displayName: 'Check version' - script: | cp NOTICE.txt $(work_dir) From fd2d286accf330fb1cfcf2f5623a3ae9007911f8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 13:42:48 +0530 Subject: [PATCH 073/513] test artifacts changes --- release-pipelines.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 62474f3dc..f7c00683d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -365,6 +365,11 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' + + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries from WD' + workingDirectory: $(root_dir) - script: | ./$(work_dir)/azcopy_linux_arm64 --version From a4b44afbcf132dac5435bfa8d554cf537cb84108 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 13:54:43 +0530 Subject: [PATCH 074/513] test artifacts changes --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index f7c00683d..f4da1b285 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -372,6 +372,7 @@ stages: workingDirectory: $(root_dir) - script: | + ./$(work_dir)/azcopy_linux_amd64 --version ./$(work_dir)/azcopy_linux_arm64 --version displayName: 'Check version' From 2e61438463465d4df6fdcf0819a945ec6192d868 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 14:08:52 +0530 Subject: [PATCH 075/513] test artifacts changes --- release-pipelines.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f4da1b285..de2c3a359 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -354,13 +354,13 @@ stages: ldd --version displayName: "GLIBC Version" - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" @@ -370,11 +370,6 @@ stages: sudo ls -lRt $(work_dir)/ displayName: 'List the binaries from WD' workingDirectory: $(root_dir) - - - script: | - ./$(work_dir)/azcopy_linux_amd64 --version - ./$(work_dir)/azcopy_linux_arm64 --version - displayName: 'Check version' - script: | cp NOTICE.txt $(work_dir) From b00da68587179343c80b08ebc5ee1cda24b923e7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 14:19:58 +0530 Subject: [PATCH 076/513] test artifacts changes --- release-pipelines.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index de2c3a359..8c03ea2cb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -309,14 +309,16 @@ stages: strategy: matrix: Linux_amd64: - imageName: 'ubuntu-20.04' - type: 'linux_amd64' - tags: 'tags' - depends: 'depends' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + imageName: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + pool: - name: "blobfuse-ubuntu-pool" + name: "blobfuse-ubn-arm64-pool" + # name: "blobfuse-ubuntu-pool" demands: - ImageOverride -equals $(agentName) @@ -361,9 +363,10 @@ stages: # - script: | # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" # displayName: 'Generate Linux AMD64 SE Integration' + # GOARCH=arm64 GOOS=linux - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' - script: | From 1c1d64e799c10333a609ddee967809218e619b73 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 14:27:17 +0530 Subject: [PATCH 077/513] test artifacts changes --- release-pipelines.yml | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8c03ea2cb..b7c54ad44 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -309,16 +309,14 @@ stages: strategy: matrix: Linux_amd64: - imageName: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - + imageName: 'ubuntu-20.04' + type: 'linux_amd64' + tags: 'tags' + depends: 'depends' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - name: "blobfuse-ubn-arm64-pool" - # name: "blobfuse-ubuntu-pool" + name: "blobfuse-ubuntu-pool" demands: - ImageOverride -equals $(agentName) @@ -356,17 +354,16 @@ stages: ldd --version displayName: "GLIBC Version" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - # GOARCH=arm64 GOOS=linux + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - script: | - go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' - script: | From 3e3466105af77c4a72318e624ad69318cd6aa8c8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 14:43:52 +0530 Subject: [PATCH 078/513] test artifacts changes --- release-pipelines.yml | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b7c54ad44..be68612bd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -387,30 +387,30 @@ stages: workingDirectory: $(root_dir) - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "AzCopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy_linux_amd64*.deb ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - # cp ./azcopy_linux_amd64*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "AzCopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy_linux_amd64*.deb ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy_linux_amd64*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy_linux_amd64*.rpm ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - # cp ./azcopy_linux_amd64*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy_linux_amd64*.rpm ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy_linux_amd64*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' # using fpm tool for packaging of our binary & performing post-install operations From 55b22f7cc257f481e4d4f29bebcd02688187bfba Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 15:55:08 +0530 Subject: [PATCH 079/513] test artifacts changes --- release-pipelines.yml | 55 ++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index be68612bd..d01f2485c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -377,13 +377,24 @@ stages: # place the generated binary files & any additional files in appropriate locations - script: | - mkdir -p pkgDir/usr/bin/ - cp -r $(work_dir) pkgDir/usr/bin/ - displayName: 'Accumulate pkg files' + mkdir -p pkgDirAmd64/usr/bin/ + cp -r $(work_dir) pkgDirAmd64/usr/bin/ + displayName: 'Accumulate amd64 pkg files' + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirArm64/usr/bin/ + cp -r $(work_dir) pkgDirArm64/usr/bin/ + displayName: 'Accumulate arm64 pkg files' + + - script: | + sudo ls -lRt pkgDirAmd64/usr/bin/ + displayName: 'List amd64 binaries' + workingDirectory: $(root_dir) - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the binaries' + sudo ls -lRt pkgDirArm64/usr/bin/ + displayName: 'List arm64 binaries' workingDirectory: $(root_dir) @@ -391,24 +402,24 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy_linux_amd64*.deb ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy_linux_amd64*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy_linux_amd64*.rpm ./azcopy_linux_amd64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy_linux_amd64*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -417,29 +428,29 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy_linux_arm64 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy_linux_arm64*.deb ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy_linux_arm64*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy_linux_arm64 --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy_linux_arm64*.rpm ./azcopy_linux_arm64-`./pkgDir/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy_linux_arm64*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy*.rpm ./azcopy_linux_arm64-`./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' - script: | - cp ./azcopy_linux_se_amd64*.rpm $(Build.ArtifactStagingDirectory) + cp ./($work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Add SE binary to signing stage' From 507b90748a9fd00b72d3e4d6bd68b99410aec8ed Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 16:06:38 +0530 Subject: [PATCH 080/513] test artifacts changes --- release-pipelines.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d01f2485c..0879f37fb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -371,22 +371,26 @@ stages: displayName: 'List the binaries from WD' workingDirectory: $(root_dir) - - script: | - cp NOTICE.txt $(work_dir) - displayName: 'Copy NOTICE.txt' - # place the generated binary files & any additional files in appropriate locations - script: | mkdir -p pkgDirAmd64/usr/bin/ - cp -r $(work_dir) pkgDirAmd64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ displayName: 'Accumulate amd64 pkg files' # place the generated binary files & any additional files in appropriate locations - script: | mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir) pkgDirArm64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ displayName: 'Accumulate arm64 pkg files' + - script: | + cp NOTICE.txt pkgDirAmd64/usr/bin/ + displayName: 'Copy NOTICE.txt' + + - script: | + cp NOTICE.txt pkgDirArm64/usr/bin/ + displayName: 'Copy NOTICE.txt' + - script: | sudo ls -lRt pkgDirAmd64/usr/bin/ displayName: 'List amd64 binaries' @@ -407,7 +411,7 @@ stages: --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' @@ -418,7 +422,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' From be4e6c23c45937d173e2cba35f989df2ff1bb296 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 16:14:50 +0530 Subject: [PATCH 081/513] test artifacts changes --- release-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0879f37fb..1a40f516b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -407,22 +407,22 @@ stages: # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azure-storage-azcopy/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -433,22 +433,22 @@ stages: # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy_linux_arm64-`./pkgDirArm64/usr/bin/azure-storage-azcopy/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy_linux_arm64-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' From 176bce80a983ee21ada278edc8cc652ac0b8b323 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 16:25:46 +0530 Subject: [PATCH 082/513] test artifacts changes --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1a40f516b..166b554e9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -411,7 +411,7 @@ stages: --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' @@ -437,7 +437,7 @@ stages: --maintainer "AzCopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' @@ -448,7 +448,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy_linux_arm64-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' From 9a07be53f2d3d014efffb56cb5ad3b9e14a1bbba Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 16:39:19 +0530 Subject: [PATCH 083/513] test artifacts changes --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 166b554e9..4da651569 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -408,7 +408,7 @@ stages: - script: | fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ + --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb @@ -434,7 +434,7 @@ stages: - script: | fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "AzCopy v-Team " \ + --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb @@ -454,7 +454,7 @@ stages: displayName: 'Make ARM64 rpm Package' - script: | - cp ./($work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp ./$(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Add SE binary to signing stage' From 264dbb90a3be251240d2ebf64bc536980d7b6346 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 16:48:55 +0530 Subject: [PATCH 084/513] test artifacts changes --- release-pipelines.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4da651569..9a0916e9b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -385,11 +385,11 @@ stages: - script: | cp NOTICE.txt pkgDirAmd64/usr/bin/ - displayName: 'Copy NOTICE.txt' + displayName: 'Copy NOTICE.txt for AMD64' - script: | cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt' + displayName: 'Copy NOTICE.txt for Arm64' - script: | sudo ls -lRt pkgDirAmd64/usr/bin/ @@ -400,6 +400,14 @@ stages: sudo ls -lRt pkgDirArm64/usr/bin/ displayName: 'List arm64 binaries' workingDirectory: $(root_dir) + + - script: | + ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + displayName: 'Check amd64 version' + + - script: | + ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + displayName: 'Check arm64 version' # using fpm tool for packaging of our binary & performing post-install operations @@ -454,7 +462,7 @@ stages: displayName: 'Make ARM64 rpm Package' - script: | - cp ./$(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Add SE binary to signing stage' From fb4ca820c4c1c53eb1b8371dc7660a163f4f3de3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 28 May 2024 17:04:06 +0530 Subject: [PATCH 085/513] test artifacts changes --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9a0916e9b..4141690be 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -406,7 +406,7 @@ stages: displayName: 'Check amd64 version' - script: | - ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version displayName: 'Check arm64 version' From 7c168e50dd107b875e96fd097a24e98c61117dfe Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 08:50:43 +0530 Subject: [PATCH 086/513] add different build stage --- release-pipelines.yml | 128 +++++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 32 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4141690be..d1d68754b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -362,10 +362,6 @@ stages: GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" displayName: 'Generate Linux AMD64 SE Integration' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' - - script: | sudo ls -lRt $(work_dir)/ displayName: 'List the binaries from WD' @@ -375,39 +371,20 @@ stages: - script: | mkdir -p pkgDirAmd64/usr/bin/ cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - displayName: 'Accumulate amd64 pkg files' - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - displayName: 'Accumulate arm64 pkg files' + displayName: 'Accumulate pkg files' - script: | cp NOTICE.txt pkgDirAmd64/usr/bin/ - displayName: 'Copy NOTICE.txt for AMD64' - - - script: | - cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt for Arm64' + displayName: 'Copy NOTICE.txt' - script: | sudo ls -lRt pkgDirAmd64/usr/bin/ - displayName: 'List amd64 binaries' - workingDirectory: $(root_dir) - - - script: | - sudo ls -lRt pkgDirArm64/usr/bin/ - displayName: 'List arm64 binaries' + displayName: 'List binaries' workingDirectory: $(root_dir) - script: | ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - displayName: 'Check amd64 version' - - - script: | - GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Check arm64 version' + displayName: 'Check version' # using fpm tool for packaging of our binary & performing post-install operations @@ -435,6 +412,98 @@ stages: workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' + - script: | + cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add SE binary to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Ubuntu + timeoutInMinutes: 120 + strategy: + matrix: + Linux_arm64: + vmImage: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + # Custom script to install Go-lang + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries from WD' + workingDirectory: $(root_dir) + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirArm64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + displayName: 'Accumulate arm64 pkg files' + + - script: | + cp NOTICE.txt pkgDirArm64/usr/bin/ + displayName: 'Copy NOTICE.txt for Arm64' + + - script: | + sudo ls -lRt pkgDirArm64/usr/bin/ + displayName: 'List arm64 binaries' + workingDirectory: $(root_dir) + + - script: | + GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + displayName: 'Check arm64 version' # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ @@ -461,11 +530,6 @@ stages: workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' - - script: | - cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add SE binary to signing stage' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' From 05f819ba49f44d573b7094a8b4d9e1c15cf830c9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 08:53:15 +0530 Subject: [PATCH 087/513] add different build stage --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d1d68754b..7d89fa9e9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -304,7 +304,7 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Ubuntu + - job: Ubuntu_Arm timeoutInMinutes: 120 strategy: matrix: From dc4f318f92fab91ed5cd7b92b2d9c1e59329d4ad Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 09:03:33 +0530 Subject: [PATCH 088/513] add different build stage --- release-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7d89fa9e9..73eda14c8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -304,7 +304,7 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Ubuntu_Arm + - job: Ubuntu timeoutInMinutes: 120 strategy: matrix: @@ -429,7 +429,7 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Ubuntu + - job: Ubuntu_Arm timeoutInMinutes: 120 strategy: matrix: @@ -451,11 +451,11 @@ stages: steps: # Custom script to install Go-lang - - task: GoTool@0 - env: - GO111MODULE: 'on' + - task: ShellScript@2 inputs: - version: $(AZCOPY_GOLANG_VERSION) + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - script: | echo 'Running GO Vet' From de0b8d85141f4812f598f0caee321e8a31f65fdc Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 10:50:05 +0530 Subject: [PATCH 089/513] add different build stage --- release-pipelines.yml | 362 +++++++++++++++++++++--------------------- 1 file changed, 181 insertions(+), 181 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 73eda14c8..6d05eece4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -391,24 +391,24 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy_linux_amd64*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy_linux_amd64*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy_linux_amd64*.rpm ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy_linux_amd64*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -429,118 +429,118 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Ubuntu_Arm - timeoutInMinutes: 120 - strategy: - matrix: - Linux_arm64: - vmImage: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' + # - job: Ubuntu_Arm + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux_arm64: + # vmImage: 'ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: 'blobfuse-ubn22-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # steps: + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries from WD' - workingDirectory: $(root_dir) + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries from WD' + # workingDirectory: $(root_dir) - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - displayName: 'Accumulate arm64 pkg files' + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirArm64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + # displayName: 'Accumulate arm64 pkg files' - - script: | - cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt for Arm64' + # - script: | + # cp NOTICE.txt pkgDirArm64/usr/bin/ + # displayName: 'Copy NOTICE.txt for Arm64' - - script: | - sudo ls -lRt pkgDirArm64/usr/bin/ - displayName: 'List arm64 binaries' - workingDirectory: $(root_dir) + # - script: | + # sudo ls -lRt pkgDirArm64/usr/bin/ + # displayName: 'List arm64 binaries' + # workingDirectory: $(root_dir) - - script: | - GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Check arm64 version' + # - script: | + # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + # displayName: 'Check arm64 version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # BuildArtifacts end here @@ -713,7 +713,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy*$(tags)*x86_64.deb + itemPattern: azCopy-signed/azcopy_linux_amd64*x86_64.deb - script: | ls -l @@ -725,24 +725,24 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy_linux_amd64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy_linux_amd64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy_linux_arm64*.deb - sudo dpkg -i azcopy_linux_arm64*.deb + sudo dpkg --info azcopy_linux_amd64*.deb + sudo dpkg -i azcopy_linux_amd64*.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) - script: | - azcopy_linux_arm64 --version + azcopy_linux_amd64 --version displayName: 'Check Version' - script: | - azcopy_linux_arm64 --help + azcopy_linux_amd64 --help displayName: 'Check Help' # # publishing the artifacts generated @@ -752,89 +752,89 @@ stages: # displayName: 'Publish Artifacts' - - job: Set_2_ARM - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22-ARM64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_2_ARM + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22-ARM64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: mount_dir + # value: '$(System.DefaultWorkingDirectory)/azcopytmp' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + # - script: | + # git clone https://github.com/Azure/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy_linux_arm64*$(tags)*arm64.deb + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-signed/azcopy_linux_arm64*$(tags)*arm64.deb - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - sudo dpkg --info azcopy_linux_arm64*.deb - sudo dpkg -i azcopy_linux_arm64*.deb - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo dpkg --info azcopy_linux_arm64*.deb + # sudo dpkg -i azcopy_linux_arm64*.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + # # # publishing the artifacts generated + # # - task: PublishBuildArtifacts@1 + # # inputs: + # # artifactName: 'blobfuse2' + # # displayName: 'Publish Artifacts' # - job: Set_3 # timeoutInMinutes: 120 From 9a8a4dfac183595230c5a7e9ee475438325a7f57 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 10:57:49 +0530 Subject: [PATCH 090/513] add different build stage --- release-pipelines.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6d05eece4..559a9ab65 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -391,24 +391,24 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy_linux_amd64 -C pkgDirAmd64/ \ + fpm -s dir -t deb -n azcopy-linux-amd64 -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy_linux_amd64*.deb ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy_linux_amd64*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy-linux-amd64*.deb ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy-linux-amd64*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy_linux_amd64 --rpm-digest sha256 -C pkgDirAmd64/ \ + fpm -s dir -t rpm -n azcopy-linux-amd64 --rpm-digest sha256 -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy_linux_amd64*.rpm ./azcopy_linux_amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy_linux_amd64*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy-linux-amd64*.rpm ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy-linux-amd64*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -713,7 +713,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy_linux_amd64*x86_64.deb + itemPattern: azCopy-signed/azcopy-linux-amd64*x86_64.deb - script: | ls -l @@ -725,14 +725,14 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy_linux_amd64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy_linux_amd64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy-linux-amd64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy-linux-amd64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy_linux_amd64*.deb - sudo dpkg -i azcopy_linux_amd64*.deb + sudo dpkg --info azcopy-linux-amd64*.deb + sudo dpkg -i azcopy-linux-amd64*.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) From a8440ed4ff26ed8a8fd68ca64089b6f5274e5d1b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 11:36:58 +0530 Subject: [PATCH 091/513] add different build stage --- release-pipelines.yml | 195 +++++++++++++++++++++--------------------- 1 file changed, 99 insertions(+), 96 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 559a9ab65..2fb31bcd0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -429,118 +429,121 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Ubuntu_Arm - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux_arm64: - # vmImage: 'ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: 'blobfuse-ubn22-arm64' + - job: Ubuntu_Arm + timeoutInMinutes: 120 + strategy: + matrix: + Linux_arm64: + vmImage: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + steps: + - script: | + sudo ls -lRt $(work_dir) + displayName: 'list' + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries from WD' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries from WD' + workingDirectory: $(root_dir) - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirArm64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - # displayName: 'Accumulate arm64 pkg files' + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirArm64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + displayName: 'Accumulate arm64 pkg files' - # - script: | - # cp NOTICE.txt pkgDirArm64/usr/bin/ - # displayName: 'Copy NOTICE.txt for Arm64' + - script: | + cp NOTICE.txt pkgDirArm64/usr/bin/ + displayName: 'Copy NOTICE.txt for Arm64' - # - script: | - # sudo ls -lRt pkgDirArm64/usr/bin/ - # displayName: 'List arm64 binaries' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt pkgDirArm64/usr/bin/ + displayName: 'List arm64 binaries' + workingDirectory: $(root_dir) - # - script: | - # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - # displayName: 'Check arm64 version' + - script: | + GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + displayName: 'Check arm64 version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # BuildArtifacts end here From 2f93b75108670382aac38e2418a40a12faf8a589 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 12:00:21 +0530 Subject: [PATCH 092/513] add different build stage --- release-pipelines.yml | 373 +++++++++++++++++++++--------------------- 1 file changed, 188 insertions(+), 185 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2fb31bcd0..01f18f80e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -451,8 +451,11 @@ stages: steps: - script: | - sudo ls -lRt $(work_dir) + chmod 777 $(work_dir) + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ displayName: 'list' + # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -919,198 +922,198 @@ stages: # TestArtifacts ends here - # - stage: ReleaseArtifacts - # dependsOn: TestArtifacts - # condition: succeeded('TestArtifacts') - # jobs: - # - job: ReleaseBlobfuse - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azcopy' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + displayName: 'List Artifacts' - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # - ${{ if eq(parameters.post_release, true) }}: - # # add release tags & push to github - # - task: GithubRelease@1 - # inputs: - # githubConnection: 'blobfuse-git-rel' - # repositoryName: 'Azure/azure-storage-fuse' - # action: 'edit' - - # target: '$(Build.SourceVersion)' - # tagSource: 'userSpecifiedTag' - - # title: ${{ parameters.tag }} - # tag: ${{ parameters.tag }} - - # assets: | - # $(Build.ArtifactStagingDirectory)/azcopy/* - # changeLogCompareToRelease: 'lastFullRelease' - # changeLogType: 'commitBased' - # isDraft: ${{ parameters.draft }} - # isPreRelease: ${{ parameters.prerelease }} - # assetUploadMode: replace - - # - ${{ if eq(parameters.publish_artifacts, true) }}: - # - stage: PublishArtifacts - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: PublishArtifacts - # timeoutInMinutes: 120 - # pool: - # vmImage: 'ubuntu-22.04' - # variables: - # - group: AZURE_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - ${{ if eq(parameters.post_release, true) }}: + # add release tags & push to github + - task: GithubRelease@1 + inputs: + githubConnection: 'blobfuse-git-rel' + repositoryName: 'Azure/azure-storage-fuse' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azcopy/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZURE_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'BlobFuse/Blobfuse' - # displayName: 'Connect to PMC artifact' - - # - script: | - # pip install pmc-cli - # displayName: 'Install pmc-cli' - - # - task: DownloadSecureFile@1 - # name: pmcCertificate - # displayName: 'Download pmc pem file' - # inputs: - # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - # - task: DownloadSecureFile@1 - # name: settings - # displayName: 'Download settings.toml file' - # inputs: - # secureFile: 'settings.toml' - - # - script: | - # pmc --version - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - # if [ $? -ne 0 ]; then - # exit 1 - # fi - # displayName: 'Test PMC installation' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Signed Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # cd mariner - # for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done - # ls -lRt - # mv blobfuse2*.rpm .. - # cd .. - # rm -r mariner/ - # displayName: 'Rename Mariner binaries' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # displayName: 'List Artifacts' - - # - script: | - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed - # displayName: 'Upload packages' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ - - # - script: | - # wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv - # cat ./packages.csv - # displayName: 'Fetch packages.csv' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ - - # - script: | - # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` - # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" - - # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` - # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" - - # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` - # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" - - # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` - # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" - - # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` - # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" - - # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` - # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" - - # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` - # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" - - # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` - # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Add uploaded packages to repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - # - script: | - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Publish the repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'BlobFuse/Blobfuse' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + ls -lRt + mv blobfuse2*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed + displayName: 'Upload packages' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` + echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + + fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` + echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" + + fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` + echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" + + fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` + echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" + + fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` + echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" + + fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` + echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" + + marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` + echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" + + marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` + echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" + + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + - script: | + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ # - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts From 1eb99b9d9db3ecebb5dcde4ba8f908ed5506c8ad Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 12:27:01 +0530 Subject: [PATCH 093/513] add different build stage --- release-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 01f18f80e..0d4d3914a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -450,12 +450,6 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - script: | - chmod 777 $(work_dir) - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -472,12 +466,18 @@ stages: # install dependencies required for compiling azcopy - script: | sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install pkg-config libsecret-1-dev wget -y sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y sudo gem install dotenv -v 2.8.1 sudo gem install fpm -V displayName: "Installing Dependencies" + - script: | + wget https://github.com/dphulkar-msft/azure-storage-azcopy/blob/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + # get glibc version with which build is done - script: | ldd --version From c03ce2ca59239b0647cc008578552883d25b8f9b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 12:37:30 +0530 Subject: [PATCH 094/513] add different build stage --- release-pipelines.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0d4d3914a..4c5d79dc0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -450,6 +450,13 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: + + - script: | + wget https://github.com/dphulkar-msft/azure-storage-azcopy/blob/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -472,12 +479,6 @@ stages: sudo gem install fpm -V displayName: "Installing Dependencies" - - script: | - wget https://github.com/dphulkar-msft/azure-storage-azcopy/blob/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - # get glibc version with which build is done - script: | ldd --version From 8e84b3290eb5c40220ff207d1728eeb1cd7ef2bd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 13:01:12 +0530 Subject: [PATCH 095/513] add different build stage --- go_installer.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go_installer.sh b/go_installer.sh index cc4b970c6..932951471 100644 --- a/go_installer.sh +++ b/go_installer.sh @@ -3,8 +3,7 @@ work_dir=$(echo $1 | sed 's:/*$::') version="1.21.8" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` -if [ $arch != "arm64" ] -then +if [ $arch != "arm64" ];then arch="amd64" fi From 2e1bcc29304b6336bd0a877395bd78ba0ac1fc42 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 13:04:02 +0530 Subject: [PATCH 096/513] add different build stage --- release-pipelines.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4c5d79dc0..f1332f3d5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -452,11 +452,13 @@ stages: steps: - script: | - wget https://github.com/dphulkar-msft/azure-storage-azcopy/blob/az-pipelineTest/go_installer.sh -P $(work_dir)/ + wget +https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh +-P $(work_dir)/ chmod 777 $(work_dir)/go_installer.sh sudo ls -lRt $(work_dir)/ displayName: 'list' - + # Custom script to install Go-lang - task: ShellScript@2 inputs: From 436e66adae985716bb7d76453e687354fff65395 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 13:06:00 +0530 Subject: [PATCH 097/513] add different build stage --- release-pipelines.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f1332f3d5..db2f7a2aa 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -452,9 +452,7 @@ stages: steps: - script: | - wget -https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh --P $(work_dir)/ + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ chmod 777 $(work_dir)/go_installer.sh sudo ls -lRt $(work_dir)/ displayName: 'list' From dd9602e70514337058f6f95491b06a0f7661a3d4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 13:21:39 +0530 Subject: [PATCH 098/513] add different build stage --- release-pipelines.yml | 163 +++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 83 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index db2f7a2aa..ed88aa69b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -311,7 +311,6 @@ stages: Linux_amd64: imageName: 'ubuntu-20.04' type: 'linux_amd64' - tags: 'tags' depends: 'depends' container: 'test-cnt-ubn-20' AgentName: "blobfuse-ubuntu20" @@ -396,7 +395,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy-linux-amd64*.deb ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + mv ./azcopy-linux-amd64*.deb ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb cp ./azcopy-linux-amd64*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' @@ -407,7 +406,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy-linux-amd64*.rpm ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + mv ./azcopy-linux-amd64*.rpm ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm cp ./azcopy-linux-amd64*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -516,24 +515,24 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + fpm -s dir -t deb -n azcopy-linux-arm64 -C pkgDirArm64/ \ -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy-linux-arm64*.deb ./azcopy-linux-arm64-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy-linux-arm64*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + fpm -s dir -t rpm -n azcopy-linux-arm64 --rpm-digest sha256 -C pkgDirArm64/ \ -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy-linux-arm64*.rpm ./azcopy-linux-arm64-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy-linux-arm64*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' @@ -680,7 +679,6 @@ stages: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' - tags: 'tags' pool: vmImage: $(vmImage) @@ -759,89 +757,89 @@ stages: # displayName: 'Publish Artifacts' - # - job: Set_2_ARM - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22-ARM64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_2_ARM + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22-ARM64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/azcopytmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: mount_dir + value: '$(System.DefaultWorkingDirectory)/azcopytmp' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-signed/azcopy_linux_arm64*$(tags)*arm64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/azcopy_linux_arm64*$(tags)*arm64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed - # - script: | - # for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/azCopy-signed + - script: | + for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed - # - script: | - # sudo dpkg --info azcopy_linux_arm64*.deb - # sudo dpkg -i azcopy_linux_arm64*.deb - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo dpkg --info azcopy_linux_arm64*.deb + sudo dpkg -i azcopy_linux_arm64*.deb + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) - # # # publishing the artifacts generated - # # - task: PublishBuildArtifacts@1 - # # inputs: - # # artifactName: 'blobfuse2' - # # displayName: 'Publish Artifacts' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'blobfuse2' + # displayName: 'Publish Artifacts' # - job: Set_3 # timeoutInMinutes: 120 @@ -851,7 +849,6 @@ stages: # agentName: "blobfuse-mariner2" # DistroVer: "Mariner2" # Description: "CBL-Mariner2 Linux" - # tags: 'tags' # container: "test-cnt-mari-2" # pool: From 395ead269895fe1d9984dbf45c4a96adaf963184 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 13:27:32 +0530 Subject: [PATCH 099/513] add different build stage --- release-pipelines.yml | 162 +++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 82 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ed88aa69b..df3d264be 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -730,7 +730,7 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy-linux-amd64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + for f in ./azcopy-linux-amd64*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; cp ./azcopy-linux-amd64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed @@ -750,11 +750,11 @@ stages: azcopy_linux_amd64 --help displayName: 'Check Help' - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy' + displayName: 'Publish Artifacts' - job: Set_2_ARM @@ -775,8 +775,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmp' - name: temp_dir value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' @@ -812,7 +810,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy_linux_arm64*$(tags)*arm64.deb + itemPattern: azCopy-signed/azcopy-linux-arm64*.deb - script: | ls -l @@ -824,22 +822,22 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy_linux_arm64*$(tags)*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy_linux_arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy-linux-arm64*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy-linux-arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy_linux_arm64*.deb - sudo dpkg -i azcopy_linux_arm64*.deb + sudo dpkg --info azcopy-linux-arm64*.deb + sudo dpkg -i azcopy-linux-arm64*.deb displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'blobfuse2' - # displayName: 'Publish Artifacts' + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy' + displayName: 'Publish Artifacts' # - job: Set_3 # timeoutInMinutes: 120 @@ -920,77 +918,77 @@ stages: # TestArtifacts ends here - - stage: ReleaseArtifacts - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseAzcopy - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - stage: ReleaseArtifacts + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') + # jobs: + # - job: ReleaseAzcopy + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + # displayName: 'List Artifacts' - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - - ${{ if eq(parameters.post_release, true) }}: - # add release tags & push to github - - task: GithubRelease@1 - inputs: - githubConnection: 'blobfuse-git-rel' - repositoryName: 'Azure/azure-storage-fuse' - action: 'edit' - - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' - - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} - - assets: | - $(Build.ArtifactStagingDirectory)/azcopy/* - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace - - - ${{ if eq(parameters.publish_artifacts, true) }}: + # - ${{ if eq(parameters.post_release, true) }}: + # # add release tags & push to github + # - task: GithubRelease@1 + # inputs: + # githubConnection: 'blobfuse-git-rel' + # repositoryName: 'Azure/azure-storage-fuse' + # action: 'edit' + + # target: '$(Build.SourceVersion)' + # tagSource: 'userSpecifiedTag' + + # title: ${{ parameters.tag }} + # tag: ${{ parameters.tag }} + + # assets: | + # $(Build.ArtifactStagingDirectory)/azcopy/* + # changeLogCompareToRelease: 'lastFullRelease' + # changeLogType: 'commitBased' + # isDraft: ${{ parameters.draft }} + # isPreRelease: ${{ parameters.prerelease }} + # assetUploadMode: replace + + # - ${{ if eq(parameters.publish_artifacts, true) }}: - stage: PublishArtifacts dependsOn: ReleaseArtifacts condition: succeeded('ReleaseArtifacts') From 0896cc818b8f55bd58f4713f5e09a718a6fdd25e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 13:31:03 +0530 Subject: [PATCH 100/513] add different build stage --- release-pipelines.yml | 242 +++++++++++++++++++++--------------------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index df3d264be..311af4da4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -988,128 +988,128 @@ stages: # isPreRelease: ${{ parameters.prerelease }} # assetUploadMode: replace - # - ${{ if eq(parameters.publish_artifacts, true) }}: - - stage: PublishArtifacts - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: PublishArtifacts - timeoutInMinutes: 120 - pool: - vmImage: 'ubuntu-22.04' - variables: - - group: AZURE_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # # - ${{ if eq(parameters.publish_artifacts, true) }}: + # - stage: PublishArtifacts + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: PublishArtifacts + # timeoutInMinutes: 120 + # pool: + # vmImage: 'ubuntu-22.04' + # variables: + # - group: AZURE_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - checkout: none - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'BlobFuse/Blobfuse' - displayName: 'Connect to PMC artifact' - - - script: | - pip install pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Signed Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - cd mariner - for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done - ls -lRt - mv blobfuse2*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - displayName: 'List Artifacts' - - - script: | - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed - displayName: 'Upload packages' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv - cat ./packages.csv - displayName: 'Fetch packages.csv' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` - echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" - - fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` - echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" - - fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` - echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" - - fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` - echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" - - fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` - echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" - - fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` - echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" - - marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` - echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" - - marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` - echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv) - displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - - script: | - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv) - displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + # steps: + # - checkout: none + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'BlobFuse/Blobfuse' + # displayName: 'Connect to PMC artifact' + + # - script: | + # pip install pmc-cli + # displayName: 'Install pmc-cli' + + # - task: DownloadSecureFile@1 + # name: pmcCertificate + # displayName: 'Download pmc pem file' + # inputs: + # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + # - task: DownloadSecureFile@1 + # name: settings + # displayName: 'Download settings.toml file' + # inputs: + # secureFile: 'settings.toml' + + # - script: | + # pmc --version + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + # if [ $? -ne 0 ]; then + # exit 1 + # fi + # displayName: 'Test PMC installation' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Signed Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # cd mariner + # for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + # ls -lRt + # mv blobfuse2*.rpm .. + # cd .. + # rm -r mariner/ + # displayName: 'Rename Mariner binaries' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # displayName: 'List Artifacts' + + # - script: | + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed + # displayName: 'Upload packages' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv + # cat ./packages.csv + # displayName: 'Fetch packages.csv' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` + # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + + # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` + # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" + + # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` + # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" + + # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` + # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" + + # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` + # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" + + # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` + # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" + + # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` + # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" + + # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` + # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" + + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Add uploaded packages to repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + + # - script: | + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ # - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts From f8af6e95c52ca725abc01acff516d6ad9fe68288 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 14:32:04 +0530 Subject: [PATCH 101/513] add different build stage --- release-pipelines.yml | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 311af4da4..266f45273 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -390,24 +390,24 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy-linux-amd64 -C pkgDirAmd64/ \ + fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy-linux-amd64*.deb ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy-linux-amd64*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy-linux-amd64 --rpm-digest sha256 -C pkgDirAmd64/ \ + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy-linux-amd64*.rpm ./azcopy-linux-amd64-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy-linux-amd64*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -515,24 +515,24 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy-linux-arm64 -C pkgDirArm64/ \ + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy-linux-arm64*.deb ./azcopy-linux-arm64-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy-linux-arm64*.deb $(Build.ArtifactStagingDirectory) + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy-linux-arm64 --rpm-digest sha256 -C pkgDirArm64/ \ + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy-linux-arm64*.rpm ./azcopy-linux-arm64-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy-linux-arm64*.rpm $(Build.ArtifactStagingDirectory) + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' @@ -730,14 +730,14 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy-linux-amd64*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy-linux-amd64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy*.deb; do mv -v "$f" "${.$(vmImage).}"; done; + cp ./azcopy-*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy-linux-amd64*.deb - sudo dpkg -i azcopy-linux-amd64*.deb + sudo dpkg --info azcopy-*.deb + sudo dpkg -i azcopy-*.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -790,6 +790,13 @@ stages: git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' workingDirectory: $(root_dir)/azure-storage-azcopy + + #TODO Removed listing + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' # Custom script to install Go-lang - task: ShellScript@2 From 10518daf148b6a589f8bbab1e45803716caf2b0d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 14:34:31 +0530 Subject: [PATCH 102/513] add different build stage --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 266f45273..d7ee0e91f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -796,7 +796,7 @@ stages: wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ chmod 777 $(work_dir)/go_installer.sh sudo ls -lRt $(work_dir)/ - displayName: 'list' + displayName: 'listing' # Custom script to install Go-lang - task: ShellScript@2 From e895ee9249329883cc79e3213a24fe9619b72a72 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 15:02:57 +0530 Subject: [PATCH 103/513] add different build stage --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d7ee0e91f..55450ecba 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -718,7 +718,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-linux-amd64*x86_64.deb + itemPattern: azCopy-signed/azcopy-*x86_64.deb - script: | ls -l @@ -817,7 +817,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-linux-arm64*.deb + itemPattern: azCopy-signed/azcopy-*.deb - script: | ls -l From 74a9e74232be2189e0a84d30bd7f214e18cdb42e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 15:52:20 +0530 Subject: [PATCH 104/513] add different build stage --- release-pipelines.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 55450ecba..b4e126fac 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -311,7 +311,7 @@ stages: Linux_amd64: imageName: 'ubuntu-20.04' type: 'linux_amd64' - depends: 'depends' + tags: 'tags' container: 'test-cnt-ubn-20' AgentName: "blobfuse-ubuntu20" pool: @@ -395,7 +395,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' @@ -406,7 +406,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -436,6 +436,7 @@ stages: vmImage: 'ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' AgentName: 'blobfuse-ubn22-arm64' + tags: 'tags' pool: name: "blobfuse-ubn-arm64-pool" @@ -520,7 +521,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' @@ -531,7 +532,7 @@ stages: --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' @@ -730,14 +731,14 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy*.deb; do mv -v "$f" "${.$(vmImage).}"; done; - cp ./azcopy-*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy*.x86-64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*x86-64.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy-*.deb - sudo dpkg -i azcopy-*.deb + sudo dpkg --info azcopy-*.x86-64.deb + sudo dpkg -i azcopy-*.x86-64.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -817,7 +818,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*.deb + itemPattern: azCopy-signed/azcopy-*arm64..deb - script: | ls -l @@ -829,14 +830,14 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy-linux-arm64*.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy-linux-arm64*$(vmImage)*.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy-linux-arm64*.deb - sudo dpkg -i azcopy-linux-arm64*.deb + sudo dpkg --info azcopy*.arm64.deb + sudo dpkg -i azcopy*.arm64.deb displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) From 5578999322a22cf87fadc0ce6d9bc489ed51d802 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 16:38:07 +0530 Subject: [PATCH 105/513] add different build stage --- release-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index b4e126fac..f36274cab 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -680,6 +680,7 @@ stages: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' + tags: 'tags' pool: vmImage: $(vmImage) @@ -766,6 +767,7 @@ stages: vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' AgentName: "blobfuse-ubn22-arm64" + tags: 'tags' pool: name: "blobfuse-ubn-arm64-pool" demands: From f5d72ba54f93aef821fdece9ca9855b4fe5c796e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 29 May 2024 16:58:13 +0530 Subject: [PATCH 106/513] add different build stage --- release-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f36274cab..2703afa3b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -720,7 +720,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*x86_64.deb + itemPattern: azCopy-signed/azcopy-*$(tags)*x86_64.deb - script: | ls -l @@ -732,14 +732,14 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy*.x86-64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*x86-64.deb $(Build.ArtifactStagingDirectory) + for f in ./azcopy*$(tags)*.x86_64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*x86_64.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy-*.x86-64.deb - sudo dpkg -i azcopy-*.x86-64.deb + sudo dpkg --info azcopy-*.x86_64.deb + sudo dpkg -i azcopy-*.x86-64_deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -820,7 +820,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*arm64..deb + itemPattern: azCopy-signed/azcopy-*$(tags)*.arm64.deb - script: | ls -l @@ -832,7 +832,7 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + for f in ./azcopy*.$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed @@ -981,7 +981,7 @@ stages: # - task: GithubRelease@1 # inputs: # githubConnection: 'blobfuse-git-rel' - # repositoryName: 'Azure/azure-storage-fuse' + # repositoryName: 'dphulkar-msft/azure-storage-azcopy' # action: 'edit' # target: '$(Build.SourceVersion)' From bf387c056c8f5d805e6452ffbc77a2859d9ed235 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 11:22:20 +0530 Subject: [PATCH 107/513] add different build stage --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2703afa3b..19147a66e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -739,7 +739,7 @@ stages: - script: | sudo dpkg --info azcopy-*.x86_64.deb - sudo dpkg -i azcopy-*.x86-64_deb + sudo dpkg -i azcopy-*.x86_64.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -832,7 +832,7 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - for f in ./azcopy*.$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + for f in ./azcopy*$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed @@ -980,7 +980,7 @@ stages: # # add release tags & push to github # - task: GithubRelease@1 # inputs: - # githubConnection: 'blobfuse-git-rel' + # githubConnection: 'dphulkar-msft' # repositoryName: 'dphulkar-msft/azure-storage-azcopy' # action: 'edit' From e359bf9a1ff64fbc6a3e8b3adbc8e413716af410 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 11:35:59 +0530 Subject: [PATCH 108/513] add release github stage --- release-pipelines.yml | 124 +++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 19147a66e..1384c80dd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -928,75 +928,75 @@ stages: # TestArtifacts ends here - # - stage: ReleaseArtifacts - # dependsOn: TestArtifacts - # condition: succeeded('TestArtifacts') - # jobs: - # - job: ReleaseAzcopy - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + displayName: 'List Artifacts' - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # - ${{ if eq(parameters.post_release, true) }}: - # # add release tags & push to github - # - task: GithubRelease@1 - # inputs: - # githubConnection: 'dphulkar-msft' - # repositoryName: 'dphulkar-msft/azure-storage-azcopy' - # action: 'edit' - - # target: '$(Build.SourceVersion)' - # tagSource: 'userSpecifiedTag' - - # title: ${{ parameters.tag }} - # tag: ${{ parameters.tag }} - - # assets: | - # $(Build.ArtifactStagingDirectory)/azcopy/* - # changeLogCompareToRelease: 'lastFullRelease' - # changeLogType: 'commitBased' - # isDraft: ${{ parameters.draft }} - # isPreRelease: ${{ parameters.prerelease }} - # assetUploadMode: replace + - ${{ if eq(parameters.post_release, true) }}: + # add release tags & push to github + - task: GithubRelease@1 + inputs: + githubConnection: 'dphulkar-msft' + repositoryName: 'dphulkar-msft/azure-storage-azcopy' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azcopy/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace # # - ${{ if eq(parameters.publish_artifacts, true) }}: # - stage: PublishArtifacts From 0581dcecf67c41c6a2b08c804c93bd4da3145f4f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 12:25:12 +0530 Subject: [PATCH 109/513] add rhel test --- release-pipelines.yml | 371 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 304 insertions(+), 67 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1384c80dd..d7a9df7d4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -672,7 +672,7 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 + - job: Set_Ubuntu_AMD64 timeoutInMinutes: 120 strategy: matrix: @@ -759,7 +759,7 @@ stages: displayName: 'Publish Artifacts' - - job: Set_2_ARM + - job: Set_Ubuntu_ARM64 timeoutInMinutes: 120 strategy: matrix: @@ -843,12 +843,249 @@ stages: displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' + # publishing the artifacts generated - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy' displayName: 'Publish Artifacts' + - job: Set_Rhel_ARM64 + timeoutInMinutes: 120 + strategy: + matrix: + RHEL-9.0: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + tags: 'tags' + container: 'test-cnt-rhel-9-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy + + #TODO Removed listing + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'listing' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/azCopy-*$(tags)*arm64.rpm + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy*$(vmImage)*.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy*$(vmImage)*.rpm + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azcopy' + displayName: 'Publish Artifacts' + + - job: Set_Rhel_AMD64 + timeoutInMinutes: 120 + strategy: + matrix: + RHEL-7.5: + agentName: "blobfuse-rhel7_5" + vmImage: 'RHEL-7.5' + tags: 'tags' + container: 'test-cnt-rhel-75' + RHEL-8.6: + agentName: "blobfuse-rhel8_6" + vmImage: 'RHEL-8.6' + container: 'test-cnt-rhel-86' + RHEL-9.0: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + tags: 'tags' + container: 'test-cnt-rhel-9' + + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + + - script: | + sudo touch /etc/yum.repos.d/centos.repo + sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' + condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) + displayName: "Update OS mirrors" + + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy + + #TODO Removed listing + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'listing' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'blobfuse2-signed' + downloadPath: $(root_dir) + itemPattern: blobfuse2-signed/azcopy*$(tags)*x86_64.rpm + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + if [ $(agentName) == "blobfuse-rhel7_5" ]; then + f=`ls ./blobfuse2*$(vmImage)*.rpm` + cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") + cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") + cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") + cp ./azcopy*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) + cp ./azcopy*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) + rm -rf ./azcopy*RHEL-7.8*.rpm + rm -rf ./azcopy*RHEL-8*.rpm + fi + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy*$(vmImage)*.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy*$(vmImage)*.rpm + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azcopy' + displayName: 'Publish Artifacts' + # - job: Set_3 # timeoutInMinutes: 120 # strategy: @@ -998,72 +1235,72 @@ stages: isPreRelease: ${{ parameters.prerelease }} assetUploadMode: replace - # # - ${{ if eq(parameters.publish_artifacts, true) }}: - # - stage: PublishArtifacts - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: PublishArtifacts - # timeoutInMinutes: 120 - # pool: - # vmImage: 'ubuntu-22.04' - # variables: - # - group: AZURE_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZURE_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'BlobFuse/Blobfuse' - # displayName: 'Connect to PMC artifact' - - # - script: | - # pip install pmc-cli - # displayName: 'Install pmc-cli' - - # - task: DownloadSecureFile@1 - # name: pmcCertificate - # displayName: 'Download pmc pem file' - # inputs: - # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - # - task: DownloadSecureFile@1 - # name: settings - # displayName: 'Download settings.toml file' - # inputs: - # secureFile: 'settings.toml' - - # - script: | - # pmc --version - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - # if [ $? -ne 0 ]; then - # exit 1 - # fi - # displayName: 'Test PMC installation' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Signed Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # cd mariner - # for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done - # ls -lRt - # mv blobfuse2*.rpm .. - # cd .. - # rm -r mariner/ - # displayName: 'Rename Mariner binaries' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # displayName: 'List Artifacts' + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'BlobFuse/Blobfuse' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + ls -lRt + mv blobfuse2*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' # - script: | # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed From 6801afb1133798b53c8cbf7e3e8aaaeb3ee56efe Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 12:26:47 +0530 Subject: [PATCH 110/513] add rhel test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d7a9df7d4..e2e7fda87 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1072,7 +1072,7 @@ stages: displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) - script: | + - script: | azcopy_linux_amd64 --version displayName: 'Check Version' From 3f69888b6fa45f474c40b058b4f64c9e468bc846 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 12:28:56 +0530 Subject: [PATCH 111/513] add rhel test --- release-pipelines.yml | 388 +++++++++++++++++++++--------------------- 1 file changed, 194 insertions(+), 194 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e2e7fda87..559f60b1a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -857,234 +857,234 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - - job: Set_Rhel_ARM64 - timeoutInMinutes: 120 - strategy: - matrix: - RHEL-9.0: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - tags: 'tags' - container: 'test-cnt-rhel-9-arm64' + # - job: Set_Rhel_ARM64 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # RHEL-9.0: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # tags: 'tags' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + # - script: | + # git clone https://github.com/Azure/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy - #TODO Removed listing - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'listing' + # #TODO Removed listing + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'listing' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-signed/azCopy-*$(tags)*arm64.rpm + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-signed/azCopy-*$(tags)*arm64.rpm - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy*$(vmImage)*.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy*$(vmImage)*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy*$(vmImage)*.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy*$(vmImage)*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - azcopy_linux_arm64 --version - displayName: 'Check Version' + # - script: | + # azcopy_linux_arm64 --version + # displayName: 'Check Version' - - script: | - azcopy_linux_arm64 --help - displayName: 'Check Help' + # - script: | + # azcopy_linux_arm64 --help + # displayName: 'Check Help' - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azcopy' - displayName: 'Publish Artifacts' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azcopy' + # displayName: 'Publish Artifacts' - - job: Set_Rhel_AMD64 - timeoutInMinutes: 120 - strategy: - matrix: - RHEL-7.5: - agentName: "blobfuse-rhel7_5" - vmImage: 'RHEL-7.5' - tags: 'tags' - container: 'test-cnt-rhel-75' - RHEL-8.6: - agentName: "blobfuse-rhel8_6" - vmImage: 'RHEL-8.6' - container: 'test-cnt-rhel-86' - RHEL-9.0: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - tags: 'tags' - container: 'test-cnt-rhel-9' + # - job: Set_Rhel_AMD64 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # RHEL-7.5: + # agentName: "blobfuse-rhel7_5" + # vmImage: 'RHEL-7.5' + # tags: 'tags' + # container: 'test-cnt-rhel-75' + # RHEL-8.6: + # agentName: "blobfuse-rhel8_6" + # vmImage: 'RHEL-8.6' + # container: 'test-cnt-rhel-86' + # RHEL-9.0: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # tags: 'tags' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - sudo touch /etc/yum.repos.d/centos.repo - sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' - condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) - displayName: "Update OS mirrors" + # - script: | + # sudo touch /etc/yum.repos.d/centos.repo + # sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' + # condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) + # displayName: "Update OS mirrors" - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + # - script: | + # git clone https://github.com/Azure/azure-storage-fuse + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy - #TODO Removed listing - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'listing' + # #TODO Removed listing + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'listing' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'blobfuse2-signed' - downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/azcopy*$(tags)*x86_64.rpm + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'blobfuse2-signed' + # downloadPath: $(root_dir) + # itemPattern: blobfuse2-signed/azcopy*$(tags)*x86_64.rpm - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - if [ $(agentName) == "blobfuse-rhel7_5" ]; then - f=`ls ./blobfuse2*$(vmImage)*.rpm` - cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") - cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") - cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") - cp ./azcopy*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) - cp ./azcopy*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) - rm -rf ./azcopy*RHEL-7.8*.rpm - rm -rf ./azcopy*RHEL-8*.rpm - fi - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + # if [ $(agentName) == "blobfuse-rhel7_5" ]; then + # f=`ls ./blobfuse2*$(vmImage)*.rpm` + # cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") + # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") + # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") + # cp ./azcopy*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) + # cp ./azcopy*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) + # rm -rf ./azcopy*RHEL-7.8*.rpm + # rm -rf ./azcopy*RHEL-8*.rpm + # fi + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy*$(vmImage)*.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy*$(vmImage)*.rpm - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy*$(vmImage)*.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy*$(vmImage)*.rpm + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' + # - script: | + # azcopy_linux_amd64 --version + # displayName: 'Check Version' - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' + # - script: | + # azcopy_linux_amd64 --help + # displayName: 'Check Help' - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azcopy' - displayName: 'Publish Artifacts' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azcopy' + # displayName: 'Publish Artifacts' # - job: Set_3 # timeoutInMinutes: 120 From afffc4f33b2514ff0b7e3a11b3fcbdcc4403d07d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 12:32:25 +0530 Subject: [PATCH 112/513] add rhel test --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 559f60b1a..2423dc33b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1254,7 +1254,7 @@ stages: - task: PipAuthenticate@1 inputs: - artifactFeeds: 'BlobFuse/Blobfuse' + artifactFeeds: 'DevExGlobalFeed' displayName: 'Connect to PMC artifact' - script: | @@ -1290,9 +1290,9 @@ stages: - script: | cd mariner - for f in ./azcopy*.rpm; do mv -v "$f" "${f/-fuse3./-cm2.}"; done + for f in ./azcopy*.rpm; do mv -v "$f" "${f/-tags./-cm2.}"; done ls -lRt - mv blobfuse2*.rpm .. + mv azcopy*.rpm .. cd .. rm -r mariner/ displayName: 'Rename Mariner binaries' From 1d2608a64eb1d427a7de990adcb72e6195190b74 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 12:34:26 +0530 Subject: [PATCH 113/513] add rhel test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2423dc33b..553cf68f5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1245,7 +1245,7 @@ stages: pool: vmImage: 'ubuntu-22.04' variables: - - group: AZURE_ESRP_SECRET + - group: AZCOPY_ESRP_SECRET - name: root_dir value: '$(System.DefaultWorkingDirectory)' From a169e5adee3d13437dd001cbfdc2feb9796eb4b9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 12:55:00 +0530 Subject: [PATCH 114/513] add rhel test --- release-pipelines.yml | 380 +++++++++++++++++++++--------------------- 1 file changed, 190 insertions(+), 190 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 553cf68f5..b85049937 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -428,126 +428,126 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Ubuntu_Arm - timeoutInMinutes: 120 - strategy: - matrix: - Linux_arm64: - vmImage: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' - tags: 'tags' + # - job: Ubuntu_Arm + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux_arm64: + # vmImage: 'ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: 'blobfuse-ubn22-arm64' + # tags: 'tags' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: + # steps: - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries from WD' - workingDirectory: $(root_dir) + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries from WD' + # workingDirectory: $(root_dir) - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - displayName: 'Accumulate arm64 pkg files' + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirArm64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + # displayName: 'Accumulate arm64 pkg files' - - script: | - cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt for Arm64' + # - script: | + # cp NOTICE.txt pkgDirArm64/usr/bin/ + # displayName: 'Copy NOTICE.txt for Arm64' - - script: | - sudo ls -lRt pkgDirArm64/usr/bin/ - displayName: 'List arm64 binaries' - workingDirectory: $(root_dir) + # - script: | + # sudo ls -lRt pkgDirArm64/usr/bin/ + # displayName: 'List arm64 binaries' + # workingDirectory: $(root_dir) - - script: | - GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Check arm64 version' + # - script: | + # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + # displayName: 'Check arm64 version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # BuildArtifacts end here @@ -759,103 +759,103 @@ stages: displayName: 'Publish Artifacts' - - job: Set_Ubuntu_ARM64 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22-ARM64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - tags: 'tags' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_Ubuntu_ARM64 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22-ARM64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # tags: 'tags' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: temp_dir + # value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + # - script: | + # git clone https://github.com/Azure/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy - #TODO Removed listing - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'listing' + # #TODO Removed listing + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'listing' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - # get glibc version with which build is done - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*$(tags)*.arm64.deb + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-signed/azcopy-*$(tags)*.arm64.deb - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy*$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed + # - script: | + # for f in ./azcopy*$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + # cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) + # displayName: 'Rename Package' + # workingDirectory: $(root_dir)/azCopy-signed - - script: | - sudo dpkg --info azcopy*.arm64.deb - sudo dpkg -i azcopy*.arm64.deb - displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo dpkg --info azcopy*.arm64.deb + # sudo dpkg -i azcopy*.arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - azcopy_linux_arm64 --version - displayName: 'Check Version' + # - script: | + # azcopy_linux_arm64 --version + # displayName: 'Check Version' - - script: | - azcopy_linux_arm64 --help - displayName: 'Check Help' + # - script: | + # azcopy_linux_arm64 --help + # displayName: 'Check Help' - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy' - displayName: 'Publish Artifacts' + # # publishing the artifacts generated + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy' + # displayName: 'Publish Artifacts' # - job: Set_Rhel_ARM64 # timeoutInMinutes: 120 @@ -1198,11 +1198,11 @@ stages: artifactName: 'azCopy-signed' downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + # displayName: 'List Artifacts' - script: | sudo apt-get update From f3c42643e70282f8832cce75d1991f5314c3c5b5 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 30 May 2024 14:31:29 +0530 Subject: [PATCH 115/513] add rhel test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b85049937..b66889f31 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1217,7 +1217,7 @@ stages: # add release tags & push to github - task: GithubRelease@1 inputs: - githubConnection: 'dphulkar-msft' + githubConnection: 'github.com_dphulkar-msft' repositoryName: 'dphulkar-msft/azure-storage-azcopy' action: 'edit' From d0276613a867c1ddc6e86090ddc90c564104bd3e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 31 May 2024 17:44:03 +0530 Subject: [PATCH 116/513] adding publishing step --- packages.csv | 3 + release-pipelines.yml | 465 +++++++++++++++++++++--------------------- 2 files changed, 234 insertions(+), 234 deletions(-) create mode 100644 packages.csv diff --git a/packages.csv b/packages.csv new file mode 100644 index 000000000..f744f8fde --- /dev/null +++ b/packages.csv @@ -0,0 +1,3 @@ +# Do not remove these comments +# Format of the file : ,,, +Ubuntu-20.04,azcopyAmdDeb,microsoft-ubuntu-focal-prod-apt,testing diff --git a/release-pipelines.yml b/release-pipelines.yml index b66889f31..142253e38 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -304,13 +304,12 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Ubuntu + - job: Ubuntu_Amd64 timeoutInMinutes: 120 strategy: matrix: Linux_amd64: imageName: 'ubuntu-20.04' - type: 'linux_amd64' tags: 'tags' container: 'test-cnt-ubn-20' AgentName: "blobfuse-ubuntu20" @@ -428,126 +427,126 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Ubuntu_Arm - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux_arm64: - # vmImage: 'ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: 'blobfuse-ubn22-arm64' - # tags: 'tags' + - job: Ubuntu_Arm64 + timeoutInMinutes: 120 + strategy: + matrix: + Linux_arm64: + vmImage: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' + tags: 'tags' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: + steps: - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries from WD' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries from WD' + workingDirectory: $(root_dir) - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirArm64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - # displayName: 'Accumulate arm64 pkg files' + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirArm64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + displayName: 'Accumulate arm64 pkg files' - # - script: | - # cp NOTICE.txt pkgDirArm64/usr/bin/ - # displayName: 'Copy NOTICE.txt for Arm64' + - script: | + cp NOTICE.txt pkgDirArm64/usr/bin/ + displayName: 'Copy NOTICE.txt for Arm64' - # - script: | - # sudo ls -lRt pkgDirArm64/usr/bin/ - # displayName: 'List arm64 binaries' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt pkgDirArm64/usr/bin/ + displayName: 'List arm64 binaries' + workingDirectory: $(root_dir) - # - script: | - # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - # displayName: 'Check arm64 version' + - script: | + GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + displayName: 'Check arm64 version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # BuildArtifacts end here @@ -676,7 +675,7 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Ubuntu-22: + Linux_amd64: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' @@ -758,104 +757,103 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - - # - job: Set_Ubuntu_ARM64 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22-ARM64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # tags: 'tags' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_Ubuntu_ARM64 + timeoutInMinutes: 120 + strategy: + matrix: + Linux_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + tags: 'tags' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: temp_dir + value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # git clone https://github.com/Azure/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # #TODO Removed listing - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'listing' + #TODO Removed listing + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'listing' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-signed/azcopy-*$(tags)*.arm64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/azcopy-*$(tags)*.arm64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed - # - script: | - # for f in ./azcopy*$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/azCopy-signed + - script: | + for f in ./azcopy*$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed - # - script: | - # sudo dpkg --info azcopy*.arm64.deb - # sudo dpkg -i azcopy*.arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo dpkg --info azcopy*.arm64.deb + sudo dpkg -i azcopy*.arm64.deb + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # azcopy_linux_arm64 --version - # displayName: 'Check Version' + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' - # - script: | - # azcopy_linux_arm64 --help - # displayName: 'Check Help' + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy' - # displayName: 'Publish Artifacts' + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy' + displayName: 'Publish Artifacts' # - job: Set_Rhel_ARM64 # timeoutInMinutes: 120 @@ -1086,7 +1084,7 @@ stages: # artifactName: 'azcopy' # displayName: 'Publish Artifacts' - # - job: Set_3 + # - job: Set_Mariner # timeoutInMinutes: 120 # strategy: # matrix: @@ -1195,14 +1193,14 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azcopy' downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + displayName: 'List Artifacts' - script: | sudo apt-get update @@ -1302,20 +1300,20 @@ stages: sudo ls -lRt $(Build.ArtifactStagingDirectory) displayName: 'List Artifacts' - # - script: | - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed - # displayName: 'Upload packages' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ + - script: | + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed + displayName: 'Upload packages' + workingDirectory: $(Build.ArtifactStagingDirectory)/ - # - script: | - # wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/packages.csv -O packages.csv - # cat ./packages.csv - # displayName: 'Fetch packages.csv' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ - # - script: | - # fuse3AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.deb` - # echo "Fuse3 AMD DEB ID: $fuse3AmdDeb" + - script: | + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*tags.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" @@ -1338,25 +1336,25 @@ stages: # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Add uploaded packages to repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ - - # - script: | - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Publish the repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/blobfuse2-signed/ + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + + - script: | + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ # - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts @@ -1366,7 +1364,6 @@ stages: # pool: # vmImage: 'ubuntu-20.04' # variables: - # - group: NightlyBlobFuse # - name: root_dir # value: '$(System.DefaultWorkingDirectory)' @@ -1383,25 +1380,25 @@ stages: # - task: DownloadBuildArtifacts@0 # displayName: 'Download Build Artifacts' # inputs: - # artifactName: 'blobfuse2' + # artifactName: 'azcopy' # downloadPath: $(Build.ArtifactStagingDirectory) # # install blobfuse2 # - script: | - # cd $(Build.ArtifactStagingDirectory)/blobfuse2 + # cd $(Build.ArtifactStagingDirectory)/azcopy # ls | grep -i ubuntu-20.04 # sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y - # blobfuse2 version - # displayName: 'Installing blobfuse2' + # azcopy version + # displayName: 'Installing azcopy' # - script: | - # wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py + # wget https://raw.githubusercontent.com/Azure/azure-storage-storage/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py # ls -l # displayName: 'Getting Python script' # workingDirectory: $(root_dir) # - script: | - # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`blobfuse2 version`" + # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy version`" # displayName: 'Updating version number' # workingDirectory: $(root_dir) From 88989e8500c7c8a7fce44637b675f8a8cf55c5f9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 31 May 2024 17:45:43 +0530 Subject: [PATCH 117/513] adding publishing step --- release-pipelines.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 142253e38..6a8addd9c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1315,27 +1315,6 @@ stages: azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*tags.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - # fuse3ArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.arm64.deb` - # echo "Fuse3 ARM DEB ID: $fuse3ArmDeb" - - # fuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.x86_64.rpm` - # echo "Fuse3 AMD RPM ID: $fuse3AmdRpm" - - # fuse3ArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse3.aarch64.rpm` - # echo "Fuse3 ARM RPM ID: $fuse3ArmRpm" - - # fuse2AmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.deb` - # echo "Fuse2 AMD DEB ID: $fuse2AmdDeb" - - # fuse2AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*fuse2.x86_64.rpm` - # echo "Fuse2 AMD RPM ID: $fuse2AmdRpm" - - # marinerFuse3AmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.x86_64.rpm` - # echo "Mariner fuse3 AMD RPM ID: $marinerFuse3AmdRpm" - - # marinerFuse3AarchRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload blobfuse2*cm2.aarch64.rpm` - # echo "Mariner fuse3 ARM RPM ID: $marinerFuse3AarchRpm" - while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName From fc1de55d1c38e7e8cde1870d541442a03a9d3303 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 31 May 2024 17:46:03 +0530 Subject: [PATCH 118/513] adding publishing step --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6a8addd9c..1e12d297e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1306,7 +1306,7 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv cat ./packages.csv displayName: 'Fetch packages.csv' workingDirectory: $(Build.ArtifactStagingDirectory)/ From 257cfce8a572a4dd6439b15775d2930840f19253 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 31 May 2024 18:14:29 +0530 Subject: [PATCH 119/513] adding publishing step --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1e12d297e..658d86493 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1193,13 +1193,13 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azcopy' + artifactName: 'azCopy' downloadPath: $(Build.ArtifactStagingDirectory) - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azcopy/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.rpm displayName: 'List Artifacts' - script: | From 3a6c02f8bf13270538aab34a1754697507b75c61 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 31 May 2024 18:41:53 +0530 Subject: [PATCH 120/513] adding publishing step --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 658d86493..23ba66435 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1199,7 +1199,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.rpm + # md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.rpm displayName: 'List Artifacts' - script: | From 83dd31f8f8a8599f7c2a8d4556212e290991e215 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 3 Jun 2024 12:48:05 +0530 Subject: [PATCH 121/513] Test artifacts for different os --- release-pipelines.yml | 604 ++++++++++++++++++++---------------------- 1 file changed, 289 insertions(+), 315 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 23ba66435..26c61ccc7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -449,7 +449,7 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - + #TODO: remove this after merging to main - script: | wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ chmod 777 $(work_dir)/go_installer.sh @@ -689,18 +689,16 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: mount_dir - value: '$(System.DefaultWorkingDirectory)/azcopy-temp' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' steps: - checkout: none + #TODO: replace or name from dphulkar-msft to Azure - script: | - git clone https://github.com/Azure/azure-storage-azcopy + git clone https://github.com/dphulkar-msft/azure-storage-azcopy displayName: 'Checkout Code' workingDirectory: $(root_dir) + - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' @@ -782,8 +780,9 @@ stages: steps: - checkout: none + # TODO: replace org name from dphulkar-msft to Azure - script: | - git clone https://github.com/Azure/azure-storage-azcopy + git clone https://github.com/dphulkar-msft/azure-storage-azcopy displayName: 'Checkout Code' workingDirectory: $(root_dir) @@ -791,13 +790,6 @@ stages: git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' workingDirectory: $(root_dir)/azure-storage-azcopy - - #TODO Removed listing - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'listing' # Custom script to install Go-lang - task: ShellScript@2 @@ -855,311 +847,293 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - # - job: Set_Rhel_ARM64 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # RHEL-9.0: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # tags: 'tags' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_Rhel_ARM64 + timeoutInMinutes: 120 + strategy: + matrix: + RHEL-9.0: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + tags: 'tags' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - checkout: none - - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy - - # #TODO Removed listing - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'listing' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-signed/azCopy-*$(tags)*arm64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-signed - - # - script: | - # for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/azCopy-signed - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy*$(vmImage)*.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy*$(vmImage)*.rpm - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - script: | - # azcopy_linux_arm64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_arm64 --help - # displayName: 'Check Help' - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azcopy' - # displayName: 'Publish Artifacts' + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/azCopy-*$(tags)*arm64.rpm + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy*$(vmImage)*.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy*$(vmImage)*.rpm + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azcopy' + displayName: 'Publish Artifacts' - # - job: Set_Rhel_AMD64 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # RHEL-7.5: - # agentName: "blobfuse-rhel7_5" - # vmImage: 'RHEL-7.5' - # tags: 'tags' - # container: 'test-cnt-rhel-75' - # RHEL-8.6: - # agentName: "blobfuse-rhel8_6" - # vmImage: 'RHEL-8.6' - # container: 'test-cnt-rhel-86' - # RHEL-9.0: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # tags: 'tags' - # container: 'test-cnt-rhel-9' + - job: Set_Rhel_AMD64 + timeoutInMinutes: 120 + strategy: + matrix: + RHEL-9.0: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + tags: 'tags' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - checkout: none - - # - script: | - # sudo touch /etc/yum.repos.d/centos.repo - # sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' - # condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) - # displayName: "Update OS mirrors" - - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-fuse - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy - - # #TODO Removed listing - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'listing' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'blobfuse2-signed' - # downloadPath: $(root_dir) - # itemPattern: blobfuse2-signed/azcopy*$(tags)*x86_64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-signed - - # - script: | - # for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - # cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - # if [ $(agentName) == "blobfuse-rhel7_5" ]; then - # f=`ls ./blobfuse2*$(vmImage)*.rpm` - # cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") - # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") - # cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") - # cp ./azcopy*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) - # cp ./azcopy*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) - # rm -rf ./azcopy*RHEL-7.8*.rpm - # rm -rf ./azcopy*RHEL-8*.rpm - # fi - # displayName: 'Rename Package' - # workingDirectory: $(root_dir)/azCopy-signed - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy*$(vmImage)*.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy*$(vmImage)*.rpm - # displayName: 'Install Package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - script: | - # azcopy_linux_amd64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_amd64 --help - # displayName: 'Check Help' - - # # publishing the artifacts generated - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azcopy' - # displayName: 'Publish Artifacts' - - # - job: Set_Mariner - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: mount_dir - # value: '$(System.DefaultWorkingDirectory)/azcopytmp' - # - name: temp_dir - # value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' - - # steps: - # - checkout: none - - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' - - # - script: | - # git clone https://github.com/Azure/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-signed/mariner/azcopy*$(tags)*x86_64.rpm - - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-signed/mariner - - # - script: | - # sudo rpm -qip azcopy*.rpm - # sudo rpm -i azcopy*.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-signed/mariner + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + + - script: | + sudo touch /etc/yum.repos.d/centos.repo + sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' + condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) + displayName: "Update OS mirrors" + + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy + + #TODO Removed listing + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'listing' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'blobfuse2-signed' + downloadPath: $(root_dir) + itemPattern: blobfuse2-signed/azcopy*$(tags)*x86_64.rpm + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; + cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) + if [ $(agentName) == "blobfuse-rhel7_5" ]; then + f=`ls ./blobfuse2*$(vmImage)*.rpm` + cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") + cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") + cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") + cp ./azcopy*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) + cp ./azcopy*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) + rm -rf ./azcopy*RHEL-7.8*.rpm + rm -rf ./azcopy*RHEL-8*.rpm + fi + displayName: 'Rename Package' + workingDirectory: $(root_dir)/azCopy-signed + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy*$(vmImage)*.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy*$(vmImage)*.rpm + displayName: 'Install Package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' + + # publishing the artifacts generated + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azcopy' + displayName: 'Publish Artifacts' + + - job: Set_Mariner + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' + + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-signed/mariner/azcopy*$(tags)*x86_64.rpm + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-signed/mariner + + - script: | + sudo rpm -qip azcopy*.rpm + sudo rpm -i azcopy*.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-signed/mariner # TestArtifacts ends here From 3d709d6000fc080f4bb70347cb4f0d1f80d2e42a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 3 Jun 2024 14:00:20 +0530 Subject: [PATCH 122/513] Test artifacts for different os --- release-pipelines.yml | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 26c61ccc7..40a3af7b7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -774,8 +774,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: temp_dir - value: '$(System.DefaultWorkingDirectory)/azcopytmpcache' steps: - checkout: none @@ -904,7 +902,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azCopy-*$(tags)*arm64.rpm + itemPattern: azCopy-signed/azcopy-*$(tags)*arm64.rpm - script: | ls -l @@ -967,19 +965,14 @@ stages: steps: - checkout: none - - script: | - sudo touch /etc/yum.repos.d/centos.repo - sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' - condition: or(eq(variables['AgentName'], 'blobfuse-rhel7_5'),eq(variables['AgentName'], 'blobfuse-rhel7_8')) - displayName: "Update OS mirrors" - - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' + #TODO: replace org name from dphulkar-msft to Azure - script: | - git clone https://github.com/Azure/azure-storage-fuse + git clone https://github.com/dphulkar-msft/azure-storage-fuse displayName: 'Checkout Code' workingDirectory: $(root_dir) @@ -988,13 +981,6 @@ stages: displayName: 'Checkout Branch' workingDirectory: $(root_dir)/azure-storage-azcopy - #TODO Removed listing - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'listing' - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -1010,9 +996,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'blobfuse2-signed' + artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: blobfuse2-signed/azcopy*$(tags)*x86_64.rpm + itemPattern: azCopy-signed/azcopy*$(tags)*x86_64.rpm - script: | ls -l @@ -1026,16 +1012,6 @@ stages: - script: | for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - if [ $(agentName) == "blobfuse-rhel7_5" ]; then - f=`ls ./blobfuse2*$(vmImage)*.rpm` - cp "$f" $(sed 's:RHEL-7.5:RHEL-7.8:' <<< "$f") - cp "$f" $(sed 's:RHEL-7.5:RHEL-8.1:' <<< "$f") - cp "$f" $(sed 's:RHEL-7.5:RHEL-8.2:' <<< "$f") - cp ./azcopy*RHEL-7.8*.rpm $(Build.ArtifactStagingDirectory) - cp ./azcopy*RHEL-8*.rpm $(Build.ArtifactStagingDirectory) - rm -rf ./azcopy*RHEL-7.8*.rpm - rm -rf ./azcopy*RHEL-8*.rpm - fi displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed From 1de1869e704eb06e233ba50e8088370022a43920 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 3 Jun 2024 15:23:49 +0530 Subject: [PATCH 123/513] Test artifacts for different os --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 40a3af7b7..8f7884679 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -972,7 +972,7 @@ stages: #TODO: replace org name from dphulkar-msft to Azure - script: | - git clone https://github.com/dphulkar-msft/azure-storage-fuse + git clone https://github.com/dphulkar-msft/azure-storage-azcopy displayName: 'Checkout Code' workingDirectory: $(root_dir) @@ -1046,6 +1046,7 @@ stages: DistroVer: "Mariner2" Description: "CBL-Mariner2 Linux" container: "test-cnt-mari-2" + tags: 'tags' pool: name: "blobfuse-mariner-pool" From 3f70f8a0437ac76e3ce70d8e4baa7d21b13f0338 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 3 Jun 2024 15:39:03 +0530 Subject: [PATCH 124/513] Test artifacts for different os --- release-pipelines.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8f7884679..241b53d18 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1150,7 +1150,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.rpm displayName: 'List Artifacts' - script: | @@ -1252,8 +1252,11 @@ stages: displayName: 'List Artifacts' - script: | - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-signed - displayName: 'Upload packages' + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.arm64.deb + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.arm64.rpm + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.x86_64.deb + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.x86_64.rpm + displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | @@ -1263,9 +1266,18 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*tags.x86_64.deb` + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + azcopyAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.rpm` + echo "Azcopy AMD DEB ID: $azcopyAmdRpm" + + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy AMD DEB ID: $azcopyArmDeb" + + azcopyArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.rpm` + echo "Azcopy AMD DEB ID: $azcopyArmRpm" + while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName From 81bf8b6d1f79c209d615329026abbdd20f7581ab Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 3 Jun 2024 16:47:39 +0530 Subject: [PATCH 125/513] Test artifacts for different os --- release-pipelines.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 241b53d18..29453100f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -590,7 +590,8 @@ stages: - script: | mkdir mariner && chmod 755 mariner - cp azCopy-temp/* mariner + cp azCopy-temp/*arm64.rpm mariner + cp azCopy-temp/*x86_64.rpm mariner sudo ls -lRt mariner md5sum mariner/* displayName: 'Copy artifacts for Mariner' @@ -938,7 +939,7 @@ stages: # publishing the artifacts generated - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azcopy' + artifactName: 'azCopy' displayName: 'Publish Artifacts' - job: Set_Rhel_AMD64 @@ -1034,7 +1035,7 @@ stages: # publishing the artifacts generated - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azcopy' + artifactName: 'azCopy' displayName: 'Publish Artifacts' - job: Set_Mariner @@ -1112,6 +1113,14 @@ stages: displayName: 'Install Package' workingDirectory: $(root_dir)/azCopy-signed/mariner + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' + # TestArtifacts ends here - stage: ReleaseArtifacts @@ -1322,7 +1331,7 @@ stages: # - task: DownloadBuildArtifacts@0 # displayName: 'Download Build Artifacts' # inputs: - # artifactName: 'azcopy' + # artifactName: 'azCopy' # downloadPath: $(Build.ArtifactStagingDirectory) # # install blobfuse2 From a9d9ba0006971170ad6bb122f24b75b2415bcaa0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 4 Jun 2024 11:34:31 +0530 Subject: [PATCH 126/513] Test artifacts for different os --- release-pipelines.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 29453100f..18a6c1084 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -522,6 +522,8 @@ stages: --description "Add description" mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + echo "Version : $version" workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' @@ -1261,10 +1263,12 @@ stages: displayName: 'List Artifacts' - script: | - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.arm64.deb - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.arm64.rpm - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.x86_64.deb - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy*.x86_64.rpm + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-cm2.x86_64.rpm + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.x86_64.rpm + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.x86_64.deb + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-cm2.arm64.rpm + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.arm64.rpm + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.arm64.deb displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory)/ From 7b1aebb6993c5919c4bab8b809add2beb56e044a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 4 Jun 2024 12:15:30 +0530 Subject: [PATCH 127/513] Test artifacts for different os --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 18a6c1084..80d17547a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -524,6 +524,7 @@ stages: cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 echo "Version : $version" + echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' @@ -1270,7 +1271,7 @@ stages: pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.arm64.rpm pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.arm64.deb displayName: 'Upload files' - workingDirectory: $(Build.ArtifactStagingDirectory)/ + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - script: | wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv From 3e023ed96ee5e119b105b47bca932f78b11d501a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 4 Jun 2024 12:28:36 +0530 Subject: [PATCH 128/513] add run ut and e2etest jobs --- release-pipelines.yml | 506 +++++++++++++++++++++--------------------- 1 file changed, 253 insertions(+), 253 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 80d17547a..e742b11df 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -43,262 +43,262 @@ trigger: none pr: none stages: - # - stage: RunTests - # jobs: - # - job: UT_On_Ubuntu - # variables: - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' + - stage: RunTests + jobs: + - job: UT_On_Ubuntu + variables: + isMutexSet: 'false' + # allow maximum build time, in case we have build congestion + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py + steps: + - task: UsePythonVersion@0 + name: 'Set_up_Python' + inputs: + versionSpec: '3.7' + + - task: GoTool@0 + name: 'Set_up_Golang' + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + - task: DownloadSecureFile@1 + name: ciGCSServiceAccountKey + displayName: 'Download GCS Service Account Key' + inputs: + secureFile: 'ci-gcs-dev.json' + + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Install dependencies' + + - script: | + pip install azure-storage-blob==12.12.0 + # set the variable to indicate that the mutex is being acquired + # note: we set it before acquiring the mutex to ensure we release the mutex. + # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # mutex but before we set this variable. + # setting this before will always work since it is valid to break an un-acquired mutex. + echo '##vso[task.setvariable variable=isMutexSet]true' + # acquire the mutex before running live tests to avoid conflicts + python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + name: 'Acquire_the_distributed_mutex' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'cmd' + coverage_name: 'cmd' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common' + coverage_name: 'common' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common/parallel' + coverage_name: 'parallel' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'ste' + coverage_name: 'ste' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'sddl' + coverage_name: 'sddl' + + - script: | + GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + go build -tags "netgo" -o test-validator ./testSuite/ + mkdir test-temp + mkdir coverage + export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + keyctl session test python ./testSuite/scripts/run.py - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o smoke_coverage.txt + echo 'Formatting coverage to json format' + $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + echo 'Formatting coverage to xml format' + $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + name: 'Run_smoke_tests' + env: + ACCOUNT_NAME: $(ACCOUNT_NAME) + ACCOUNT_KEY: $(ACCOUNT_KEY) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + FILESYSTEM_URL: $(FILESYSTEM_URL) + OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + SHARE_SAS_URL: $(SHARE_SAS_URL) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + condition: succeededOrFailed() + # Smoke Tests Publishing + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + - script: | + pip install azure-storage-blob==12.12.0 + python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + name: 'Release_the_distributed_mutex' + # this runs even if the job was canceled (only if the mutex was acquired by this job) + condition: and(always(), eq(variables['isMutexSet'], 'true')) + - job: E2E_Test + timeoutInMinutes: 360 + # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + build_name: 'azcopy_linux_amd64' + display_name: "Linux" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + Windows: + imageName: 'windows-2019' + build_name: 'azcopy_windows_amd64.exe' + display_name: "Windows" + type: 'windows' + go_path: 'C:\Users\VssAdministrator\go\bin\' + suffix: '.exe' + run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + output_test_logs: 'cat test.txt' + save_exit_code: 'set exit_code=%errorlevel%' + return_exit_code: 'exit %exit_code%' + go_build: 'go build -cover -o $(build_name)' + MacOS: + imageName: 'macos-11' + build_name: 'azcopy_darwin_amd64' + display_name: "MacOS" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + pool: + vmImage: $(imageName) - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + steps: + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + pwsh: 'true' + displayName: 'Install Powershell Az Module' + - task: GoTool@0 + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Installing dependencies' + - bash: | + echo "##vso[task.setvariable variable=CGO_ENABLED]0" + displayName: 'Set CGO_ENABLED for Windows' + condition: eq(variables.type, 'windows') + - bash: | + npm install -g azurite + mkdir azurite + azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + # Running E2E Tests on AMD64 + - script: | + mkdir coverage + echo 'Building executable' + $(go_build) + echo 'Running tests' + $(run_e2e) + $(save_exit_code) + echo "Generating junit report" + cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + $(output_test_logs) + echo 'Formatting coverage to json format' + $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + echo 'Formatting coverage to xml format' + $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + $(return_exit_code) + env: + AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + displayName: 'E2E Test $(display_name) - AMD64' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish logs' + condition: succeededOrFailed() + inputs: + pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + artifactName: logs + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + testRunTitle: 'Go on $(display_name)' + + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts #dependsOn: RunTests From 8f43c903154bf2942d358572bc8d6d83a71f7204 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 11:20:01 +0530 Subject: [PATCH 129/513] test workflow --- release-pipelines.yml | 563 +++++++++++++++++++++--------------------- 1 file changed, 276 insertions(+), 287 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e742b11df..b84c232c6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -43,272 +43,272 @@ trigger: none pr: none stages: - - stage: RunTests - jobs: - - job: UT_On_Ubuntu - variables: - isMutexSet: 'false' - # allow maximum build time, in case we have build congestion - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' - - steps: - - task: UsePythonVersion@0 - name: 'Set_up_Python' - inputs: - versionSpec: '3.7' - - - task: GoTool@0 - name: 'Set_up_Golang' - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - - task: DownloadSecureFile@1 - name: ciGCSServiceAccountKey - displayName: 'Download GCS Service Account Key' - inputs: - secureFile: 'ci-gcs-dev.json' - - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Install dependencies' - - - script: | - pip install azure-storage-blob==12.12.0 - # set the variable to indicate that the mutex is being acquired - # note: we set it before acquiring the mutex to ensure we release the mutex. - # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # mutex but before we set this variable. - # setting this before will always work since it is valid to break an un-acquired mutex. - echo '##vso[task.setvariable variable=isMutexSet]true' - # acquire the mutex before running live tests to avoid conflicts - python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - name: 'Acquire_the_distributed_mutex' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'cmd' - coverage_name: 'cmd' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common' - coverage_name: 'common' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common/parallel' - coverage_name: 'parallel' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'ste' - coverage_name: 'ste' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'sddl' - coverage_name: 'sddl' + # - stage: RunTests + # jobs: + # - job: UT_On_Ubuntu + # variables: + # isMutexSet: 'false' + # # allow maximum build time, in case we have build congestion + # timeoutInMinutes: 360 + # pool: + # vmImage: 'ubuntu-20.04' - - script: | - GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - go build -tags "netgo" -o test-validator ./testSuite/ - mkdir test-temp - mkdir coverage - export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - keyctl session test python ./testSuite/scripts/run.py + # steps: + # - task: UsePythonVersion@0 + # name: 'Set_up_Python' + # inputs: + # versionSpec: '3.7' + + # - task: GoTool@0 + # name: 'Set_up_Golang' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + # - task: DownloadSecureFile@1 + # name: ciGCSServiceAccountKey + # displayName: 'Download GCS Service Account Key' + # inputs: + # secureFile: 'ci-gcs-dev.json' + + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Install dependencies' + + # - script: | + # pip install azure-storage-blob==12.12.0 + # # set the variable to indicate that the mutex is being acquired + # # note: we set it before acquiring the mutex to ensure we release the mutex. + # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # # mutex but before we set this variable. + # # setting this before will always work since it is valid to break an un-acquired mutex. + # echo '##vso[task.setvariable variable=isMutexSet]true' + # # acquire the mutex before running live tests to avoid conflicts + # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + # name: 'Acquire_the_distributed_mutex' + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'cmd' + # coverage_name: 'cmd' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common' + # coverage_name: 'common' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common/parallel' + # coverage_name: 'parallel' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'ste' + # coverage_name: 'ste' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'sddl' + # coverage_name: 'sddl' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + # go build -tags "netgo" -o test-validator ./testSuite/ + # mkdir test-temp + # mkdir coverage + # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + # export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + # keyctl session test python ./testSuite/scripts/run.py - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o smoke_coverage.txt - echo 'Formatting coverage to json format' - $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - echo 'Formatting coverage to xml format' - $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - name: 'Run_smoke_tests' - env: - ACCOUNT_NAME: $(ACCOUNT_NAME) - ACCOUNT_KEY: $(ACCOUNT_KEY) - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - FILESYSTEM_URL: $(FILESYSTEM_URL) - OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - SHARE_SAS_URL: $(SHARE_SAS_URL) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - condition: succeededOrFailed() - # Smoke Tests Publishing - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - - script: | - pip install azure-storage-blob==12.12.0 - python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - name: 'Release_the_distributed_mutex' - # this runs even if the job was canceled (only if the mutex was acquired by this job) - condition: and(always(), eq(variables['isMutexSet'], 'true')) - - job: E2E_Test - timeoutInMinutes: 360 - # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - strategy: - matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - build_name: 'azcopy_linux_amd64' - display_name: "Linux" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - Windows: - imageName: 'windows-2019' - build_name: 'azcopy_windows_amd64.exe' - display_name: "Windows" - type: 'windows' - go_path: 'C:\Users\VssAdministrator\go\bin\' - suffix: '.exe' - run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - output_test_logs: 'cat test.txt' - save_exit_code: 'set exit_code=%errorlevel%' - return_exit_code: 'exit %exit_code%' - go_build: 'go build -cover -o $(build_name)' - MacOS: - imageName: 'macos-11' - build_name: 'azcopy_darwin_amd64' - display_name: "MacOS" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - pool: - vmImage: $(imageName) - - steps: - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - pwsh: 'true' - displayName: 'Install Powershell Az Module' - - task: GoTool@0 - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Installing dependencies' - - bash: | - echo "##vso[task.setvariable variable=CGO_ENABLED]0" - displayName: 'Set CGO_ENABLED for Windows' - condition: eq(variables.type, 'windows') - - bash: | - npm install -g azurite - mkdir azurite - azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' - # Running E2E Tests on AMD64 - - script: | - mkdir coverage - echo 'Building executable' - $(go_build) - echo 'Running tests' - $(run_e2e) - $(save_exit_code) - echo "Generating junit report" - cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - $(output_test_logs) - echo 'Formatting coverage to json format' - $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - echo 'Formatting coverage to xml format' - $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - $(return_exit_code) - env: - AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - displayName: 'E2E Test $(display_name) - AMD64' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish logs' - condition: succeededOrFailed() - inputs: - pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - artifactName: logs - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testRunner: JUnit - testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - testRunTitle: 'Go on $(display_name)' + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o smoke_coverage.txt + # echo 'Formatting coverage to json format' + # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + # echo 'Formatting coverage to xml format' + # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + # name: 'Run_smoke_tests' + # env: + # ACCOUNT_NAME: $(ACCOUNT_NAME) + # ACCOUNT_KEY: $(ACCOUNT_KEY) + # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + # FILESYSTEM_URL: $(FILESYSTEM_URL) + # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + # SHARE_SAS_URL: $(SHARE_SAS_URL) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # condition: succeededOrFailed() + # # Smoke Tests Publishing + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + # - script: | + # pip install azure-storage-blob==12.12.0 + # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + # name: 'Release_the_distributed_mutex' + # # this runs even if the job was canceled (only if the mutex was acquired by this job) + # condition: and(always(), eq(variables['isMutexSet'], 'true')) + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' + + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs + + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' + + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Ubuntu_Amd64 + - job: Set_1 timeoutInMinutes: 120 strategy: matrix: - Linux_amd64: + Ubuntu_amd64: imageName: 'ubuntu-20.04' tags: 'tags' container: 'test-cnt-ubn-20' @@ -362,7 +362,7 @@ stages: - script: | sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries from WD' + displayName: 'List the binaries' workingDirectory: $(root_dir) # place the generated binary files & any additional files in appropriate locations @@ -375,14 +375,9 @@ stages: cp NOTICE.txt pkgDirAmd64/usr/bin/ displayName: 'Copy NOTICE.txt' - - script: | - sudo ls -lRt pkgDirAmd64/usr/bin/ - displayName: 'List binaries' - workingDirectory: $(root_dir) - - script: | ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - displayName: 'Check version' + displayName: 'Verify the generated build' # using fpm tool for packaging of our binary & performing post-install operations @@ -427,11 +422,11 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Ubuntu_Arm64 + - job: Set_2 timeoutInMinutes: 120 strategy: matrix: - Linux_arm64: + Ubuntu_arm64: vmImage: 'ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' AgentName: 'blobfuse-ubn22-arm64' @@ -489,7 +484,7 @@ stages: - script: | sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries from WD' + displayName: 'List the binaries' workingDirectory: $(root_dir) # place the generated binary files & any additional files in appropriate locations @@ -500,16 +495,11 @@ stages: - script: | cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt for Arm64' - - - script: | - sudo ls -lRt pkgDirArm64/usr/bin/ - displayName: 'List arm64 binaries' - workingDirectory: $(root_dir) + displayName: 'Copy NOTICE.txt' - script: | GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Check arm64 version' + displayName: 'Verify the generated build' # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ @@ -675,11 +665,11 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_Ubuntu_AMD64 + - job: Set_1 timeoutInMinutes: 120 strategy: matrix: - Linux_amd64: + Ubuntu_amd64: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' @@ -697,7 +687,7 @@ stages: steps: - checkout: none - #TODO: replace or name from dphulkar-msft to Azure + #TODO: replace org name from dphulkar-msft to Azure - script: | git clone https://github.com/dphulkar-msft/azure-storage-azcopy displayName: 'Checkout Code' @@ -709,12 +699,11 @@ stages: workingDirectory: $(root_dir)/azure-storage-azcopy # Custom script to install Go-lang - - task: GoTool@0 - env: - GO111MODULE: 'on' + - task: ShellScript@2 inputs: - version: $(AZCOPY_GOLANG_VERSION) - displayName: "Install Go" + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' @@ -759,11 +748,11 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - - job: Set_Ubuntu_ARM64 + - job: Set_2 timeoutInMinutes: 120 strategy: matrix: - Linux_arm64: + Ubuntu_arm64: vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' AgentName: "blobfuse-ubn22-arm64" @@ -849,11 +838,11 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - - job: Set_Rhel_ARM64 + - job: Set_3 timeoutInMinutes: 120 strategy: matrix: - RHEL-9.0: + Rhel_Arm64: agentName: "blobfuse-rhel9-arm64" vmImage: 'RHEL-9.0' tags: 'tags' @@ -945,11 +934,11 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - - job: Set_Rhel_AMD64 + - job: Set_4 timeoutInMinutes: 120 strategy: matrix: - RHEL-9.0: + Rhel_Amd64: agentName: "blobfuse-rhel9" vmImage: 'RHEL-9.0' tags: 'tags' @@ -1041,7 +1030,7 @@ stages: artifactName: 'azCopy' displayName: 'Publish Artifacts' - - job: Set_Mariner + - job: Set_5 timeoutInMinutes: 120 strategy: matrix: From 5d1cbe1b223ad5e45346453fc84c091825eb8dde Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 16:45:09 +0530 Subject: [PATCH 130/513] test workflow --- release-pipelines.yml | 414 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 408 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b84c232c6..1d280bff6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -325,12 +325,19 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + # Custom script to install Go-lang - - task: GoTool@0 - env: - GO111MODULE: 'on' + - task: ShellScript@2 inputs: - version: $(AZCOPY_GOLANG_VERSION) + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - script: | echo 'Running GO Vet' @@ -540,6 +547,225 @@ stages: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)/azcopy_windows_amd64' + - name: windows_386 + value: '$(work_dir)/azcopy_windows_386' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + mkdir $(windows_amd64) + mkdir $(windows_386) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + go build -o "$(windows_amd64)/azcopy_windows_amd64.exe" + raw_version_output=(`$(windows_amd64)/azcopy_windows_amd64.exe --version`) + azcopy_version=${raw_version_output[2]} + echo "Version : $azcopy_version" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(windows_386)/azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(work_dir)/azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + - script: | + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_386) + displayName: 'Copy Notice file' + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(work_dir)/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(root_dir) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_amd64" + raw_version_output=(`$(work_dir)/azcopy_darwin_amd64/azcopy_darwin_amd64 --version`) + azcopy_version=${raw_version_output[2]} + echo "Version : $azcopy_version" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_arm64" + displayName: 'Generate MacOS Build with Arm64' + + - script: | + sudo ls -lRt $(work_dir) + displayName: 'List the amd64 files' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + displayName: 'Copy Notice file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # BuildArtifacts end here @@ -547,7 +773,120 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - - job: SignAzCopy + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-temp/*arm64.rpm mariner + cp azCopy-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 timeoutInMinutes: 120 strategy: matrix: @@ -638,6 +977,68 @@ stages: } ] + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy windows' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/windows' + Pattern: '*.exe' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_WIN_KEY_CODE)", + "OperationSetCode": "SigntoolSign", + "Parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "ToolName": "signtool.exe", + "ToolVersion": "6.2.9304.0" + + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy macos' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mac' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MAC_KEY_CODE)", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* @@ -659,6 +1060,7 @@ stages: inputs: artifactName: 'azCopy-signed' displayName: 'Publish Signed Artifacts' + # SignArtifacts end here - stage: TestArtifacts @@ -1178,7 +1580,7 @@ stages: tag: ${{ parameters.tag }} assets: | - $(Build.ArtifactStagingDirectory)/azcopy/* + $(Build.ArtifactStagingDirectory)/azCopy/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} From ed27f446aedc58e055880335fb855d4db1865b33 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 16:56:38 +0530 Subject: [PATCH 131/513] test workflow --- release-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1d280bff6..ab2eef6f3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1029,13 +1029,13 @@ stages: inlineOperation: | [ { - "KeyCode" : "$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" + "KeyCode" : "$(ESRP_AZCOPY_MAC_KEY_CODE)", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" } ] From acbce94768e35f24d0fae67c99f24f0e05425959 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 17:01:28 +0530 Subject: [PATCH 132/513] test workflow --- release-pipelines.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ab2eef6f3..ce6b93712 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1027,17 +1027,17 @@ stages: signConfigType: inlineSignParams VerboseLogin: true inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MAC_KEY_CODE)", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] # Validate signed images have md5sum changed - script: | From 218b8306db56a25da03c2155a9612bede83d6943 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 17:11:51 +0530 Subject: [PATCH 133/513] test workflow --- release-pipelines.yml | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ce6b93712..35c947cb8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -571,19 +571,11 @@ stages: value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 + - task: GoTool@0 + env: + GO111MODULE: 'on' inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + version: $(AZCOPY_GOLANG_VERSION) - script: | echo 'Running GO Vet' @@ -690,19 +682,11 @@ stages: value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 + - task: GoTool@0 + env: + GO111MODULE: 'on' inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + version: $(AZCOPY_GOLANG_VERSION) - script: | echo 'Running GO Vet' From 2c33f7880b81e6b1d62f83834f2194a9977f0fbb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 17:38:31 +0530 Subject: [PATCH 134/513] test workflow --- release-pipelines.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 35c947cb8..bbb1419cd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -562,13 +562,13 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - name: windows_amd64 - value: '$(work_dir)/azcopy_windows_amd64' + value: '$(work_dir)\azcopy_windows_amd64' - name: windows_386 - value: '$(work_dir)/azcopy_windows_386' + value: '$(work_dir)\azcopy_windows_386' - name: archives - value: $(root_dir)/archives + value: $(root_dir)\archives steps: - task: GoTool@0 @@ -590,8 +590,8 @@ stages: displayName: 'Create required directories' - script: | - go build -o "$(windows_amd64)/azcopy_windows_amd64.exe" - raw_version_output=(`$(windows_amd64)/azcopy_windows_amd64.exe --version`) + go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + raw_version_output=(`$(windows_amd64)\azcopy_windows_amd64.exe --version`) azcopy_version=${raw_version_output[2]} echo "Version : $azcopy_version" displayName: 'Generate Windows AMD64' @@ -601,7 +601,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(windows_386)/azcopy_windows_386.exe" + go build -o "$(windows_386)\azcopy_windows_386.exe" displayName: 'Generate Windows i386' env: GOARCH: 386 @@ -609,7 +609,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(work_dir)/azcopy_windows_v7_arm.exe" + go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" displayName: 'Generate Windows ARM' env: GOARCH: arm @@ -618,7 +618,7 @@ stages: CGO_ENABLED: 0 - script: | - sudo ls -lRt $(work_dir)/ + sudo ls -lRt $(work_dir) displayName: 'List the binaries' workingDirectory: $(root_dir) From a10013764fb02b146e29ec8f406a30481a11590b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 17:40:43 +0530 Subject: [PATCH 135/513] test workflow --- release-pipelines.yml | 474 +++++++++++++++++++++--------------------- 1 file changed, 237 insertions(+), 237 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index bbb1419cd..6443b9c6e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -304,249 +304,249 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - tags: 'tags' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirAmd64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - displayName: 'Accumulate pkg files' - - - script: | - cp NOTICE.txt pkgDirAmd64/usr/bin/ - displayName: 'Copy NOTICE.txt' - - - script: | - ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - displayName: 'Verify the generated build' + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # tags: 'tags' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirAmd64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ + # displayName: 'Accumulate pkg files' + + # - script: | + # cp NOTICE.txt pkgDirAmd64/usr/bin/ + # displayName: 'Copy NOTICE.txt' + + # - script: | + # ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + # displayName: 'Verify the generated build' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - script: | - cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add SE binary to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - script: | + # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add SE binary to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' - tags: 'tags' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' - - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - displayName: 'Accumulate arm64 pkg files' - - - script: | - cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt' - - - script: | - GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Verify the generated build' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - echo "Version : $version" - echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: 'blobfuse-ubn22-arm64' + # tags: 'tags' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' + + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirArm64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + # displayName: 'Accumulate arm64 pkg files' + + # - script: | + # cp NOTICE.txt pkgDirArm64/usr/bin/ + # displayName: 'Copy NOTICE.txt' + + # - script: | + # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + # displayName: 'Verify the generated build' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + # echo "Version : $version" + # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "Add description" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 From 1251b5947949440c6fa7bc752016576a1296f8fc Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 17:51:37 +0530 Subject: [PATCH 136/513] test windows and mac buildworkflow --- release-pipelines.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6443b9c6e..32e7ba24a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -591,8 +591,9 @@ stages: - script: | go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - raw_version_output=(`$(windows_amd64)\azcopy_windows_amd64.exe --version`) - azcopy_version=${raw_version_output[2]} + $raw_version_output = & '$(windows_amd64)\azcopy_windows_amd64.exe' --version + $raw_version_output_array = $raw_version_output -split "`n" + $azcopy_version = $raw_version_output_array[2] echo "Version : $azcopy_version" displayName: 'Generate Windows AMD64' env: @@ -702,9 +703,11 @@ stages: - script: | CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_amd64" - raw_version_output=(`$(work_dir)/azcopy_darwin_amd64/azcopy_darwin_amd64 --version`) - azcopy_version=${raw_version_output[2]} - echo "Version : $azcopy_version" + + raw_version_output=$("$(windows_amd64)/azcopy_windows_amd64.exe" --version) + IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" + azcopy_version=${raw_version_output_array[2]} + echo "AzCopy Version: $azcopy_version" displayName: 'Generate MacOS Build with AMD64' - script: | From 26752dfe84bef66d2c0dff97a3ecbb167e0675a5 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 18:20:36 +0530 Subject: [PATCH 137/513] test windows and mac buildworkflow --- release-pipelines.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 32e7ba24a..6c99c7f59 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -704,10 +704,13 @@ stages: - script: | CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_amd64" - raw_version_output=$("$(windows_amd64)/azcopy_windows_amd64.exe" --version) - IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" - azcopy_version=${raw_version_output_array[2]} - echo "AzCopy Version: $azcopy_version" + for /f "tokens=1-3 delims= " %%a in ('"$(windows_amd64)\azcopy_windows_amd64.exe" --version') do ( + set raw_version_output=%%a %%b %%c + ) + for /f "tokens=3 delims= " %%a in ("%raw_version_output%") do ( + set azcopy_version=%%a + ) + echo AzCopy Version: %azcopy_version% displayName: 'Generate MacOS Build with AMD64' - script: | @@ -727,19 +730,19 @@ stages: displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + archiveFile: '$(archives)/azcopy_darwin_amd64_%azcopy_version%.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + archiveFile: '$(archives)/azcopy_darwin_arm64_%azcopy_version%.zip' continueOnError: true - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_amd64_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 From e6a64ec255a2c87ceabeb183cdd5f0e99fac79c1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 18:31:54 +0530 Subject: [PATCH 138/513] test windows and mac buildworkflow --- release-pipelines.yml | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6c99c7f59..f38a5b05c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -591,10 +591,13 @@ stages: - script: | go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - $raw_version_output = & '$(windows_amd64)\azcopy_windows_amd64.exe' --version - $raw_version_output_array = $raw_version_output -split "`n" - $azcopy_version = $raw_version_output_array[2] - echo "Version : $azcopy_version" + for /f "tokens=1-3 delims= " %%a in ('"$(windows_amd64)\azcopy_windows_amd64.exe" --version') do ( + set raw_version_output=%%a %%b %%c + ) + for /f "tokens=3 delims= " %%a in ("%raw_version_output%") do ( + set azcopy_version=%%a + ) + echo AzCopy Version: %azcopy_version% displayName: 'Generate Windows AMD64' env: GOARCH: amd64 @@ -632,19 +635,19 @@ stages: displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + archiveFile: '$(archives)/azcopy_windows_amd64_%azcopy_version%.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + archiveFile: '$(archives)/azcopy_windows_386_%azcopy_version%.zip' continueOnError: true - script: | - cp $(archives)/azcopy_windows_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_amd64_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) cp $(work_dir)/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' @@ -704,13 +707,11 @@ stages: - script: | CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_amd64" - for /f "tokens=1-3 delims= " %%a in ('"$(windows_amd64)\azcopy_windows_amd64.exe" --version') do ( - set raw_version_output=%%a %%b %%c - ) - for /f "tokens=3 delims= " %%a in ("%raw_version_output%") do ( - set azcopy_version=%%a - ) - echo AzCopy Version: %azcopy_version% + raw_version_output=$("$(windows_amd64)/azcopy_windows_amd64.exe" --version) + IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" + + azcopy_version=${raw_version_output_array[2]} + echo "AzCopy Version: $azcopy_version" displayName: 'Generate MacOS Build with AMD64' - script: | @@ -730,19 +731,19 @@ stages: displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_%azcopy_version%.zip' + archiveFile: '$(archives)/azcopy_darwin_amd64_$azcopy_version.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_%azcopy_version%.zip' + archiveFile: '$(archives)/azcopy_darwin_arm64_$azcopy_version.zip' continueOnError: true - script: | - cp $(archives)/azcopy_darwin_amd64_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_amd64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 From 3f85a1266046c06fd1a50fced87f3604876c3472 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 18:51:45 +0530 Subject: [PATCH 139/513] test windows and mac buildworkflow --- release-pipelines.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f38a5b05c..da4d4743d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -622,9 +622,10 @@ stages: CGO_ENABLED: 0 - script: | - sudo ls -lRt $(work_dir) + dir $(work_dir) + dir $(windows_386) + dir $(windows_amd64) displayName: 'List the binaries' - workingDirectory: $(root_dir) - script: | cp NOTICE.txt $(windows_amd64) @@ -705,9 +706,9 @@ stages: displayName: 'Create required directories' - script: | - CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_amd64" + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - raw_version_output=$("$(windows_amd64)/azcopy_windows_amd64.exe" --version) + raw_version_output=$("$(darwin_amd_dir)/azcopy_darwin_amd64" --version) IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" azcopy_version=${raw_version_output_array[2]} From d9b663f3a4b47670852508f634f14fba54edfd0c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 5 Jun 2024 19:09:32 +0530 Subject: [PATCH 140/513] test windows and mac buildworkflow --- release-pipelines.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index da4d4743d..0b7ed6b1f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -636,20 +636,20 @@ stages: displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_%azcopy_version%.zip' + archiveFile: '$(archives)/azcopy_windows_amd64*.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_%azcopy_version%.zip' + archiveFile: '$(archives)/azcopy_windows_386*.zip' continueOnError: true - script: | - cp $(archives)/azcopy_windows_amd64_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_%azcopy_version%.zip $(Build.ArtifactStagingDirectory) - cp $(work_dir)/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) + copy "%archives%\azcopy_windows_amd64*.zip" "%Build.ArtifactStagingDirectory%" + copy "%archives%\azcopy_windows_386*.zip" "%Build.ArtifactStagingDirectory%" + copy %work_dir%/azcopy_windows_v7_arm.exe "%Build.ArtifactStagingDirectory% displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 @@ -711,6 +711,11 @@ stages: raw_version_output=$("$(darwin_amd_dir)/azcopy_darwin_amd64" --version) IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" + for element in "${raw_version_output_array[@]}"; do + echo "Array Element: $element" + done + + echo "Array Length: ${#raw_version_output_array[@]}" azcopy_version=${raw_version_output_array[2]} echo "AzCopy Version: $azcopy_version" displayName: 'Generate MacOS Build with AMD64' From 3b8c0cf9f76c89c6c2aea363d60943ed54953e8c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 13:57:46 +0530 Subject: [PATCH 141/513] test windows and mac buildworkflow --- release-pipelines.yml | 589 +++++++++++++++++++++++++++--------------- 1 file changed, 374 insertions(+), 215 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0b7ed6b1f..aa8914ef0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1,6 +1,7 @@ variables: AZCOPY_GOLANG_VERSION: '1.21.8' AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' + AZCOPY_VERSION: '0.0.0' parameters: - name: tag @@ -304,17 +305,146 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - # - job: Set_1 + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + tags: 'tags' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirAmd64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ + displayName: 'Accumulate pkg files' + + - script: | + cp NOTICE.txt pkgDirAmd64/usr/bin/ + displayName: 'Copy NOTICE.txt' + + - script: | + ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) + azcopy_version=${raw_version_output[2]} + echo "Version : $azcopy_version" + displayName: 'Verify the generated build' + + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "Add description" + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - script: | + cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add SE binary to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + # - job: Set_2 # timeoutInMinutes: 120 # strategy: # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' + # Ubuntu_arm64: + # vmImage: 'ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: 'blobfuse-ubn22-arm64' # tags: 'tags' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" + # pool: - # name: "blobfuse-ubuntu-pool" + # name: "blobfuse-ubn-arm64-pool" # demands: # - ImageOverride -equals $(agentName) @@ -348,7 +478,7 @@ stages: # # install dependencies required for compiling azcopy # - script: | # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install pkg-config libsecret-1-dev wget -y # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y # sudo gem install dotenv -v 2.8.1 # sudo gem install fpm -V @@ -360,12 +490,8 @@ stages: # displayName: "GLIBC Version" # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' # - script: | # sudo ls -lRt $(work_dir)/ @@ -374,48 +500,161 @@ stages: # # place the generated binary files & any additional files in appropriate locations # - script: | - # mkdir -p pkgDirAmd64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - # displayName: 'Accumulate pkg files' + # mkdir -p pkgDirArm64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + # displayName: 'Accumulate arm64 pkg files' # - script: | - # cp NOTICE.txt pkgDirAmd64/usr/bin/ + # cp NOTICE.txt pkgDirArm64/usr/bin/ # displayName: 'Copy NOTICE.txt' # - script: | - # ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version # displayName: 'Verify the generated build' - - + # # using fpm tool for packaging of our binary & performing post-install operations # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # # TODO: Add description # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ # --maintainer "Azcopy v-Team " \ # --url "https://github.com/Azure/azure-storage-azcopy" \ # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + # echo "Version : $version" + # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # displayName: 'Make ARM64 deb Package' # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ # --maintainer "Azcopy v-Team " \ # --url "https://github.com/Azure/azure-storage-azcopy" \ # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' + # displayName: 'Make ARM64 rpm Package' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() + + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + # - name: windows_amd64 + # value: '$(work_dir)\azcopy_windows_amd64' + # - name: windows_386 + # value: '$(work_dir)\azcopy_windows_386' + # - name: archives + # value: $(root_dir)\archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) # - script: | - # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' # workingDirectory: $(root_dir) - # displayName: 'Add SE binary to signing stage' + + # - script: | + # mkdir $(windows_amd64) + # mkdir $(windows_386) + # mkdir $(archives) + # displayName: 'Create required directories' + + # - script: | + # go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + # for /f "tokens=1-3 delims= " %%a in ('"$(windows_amd64)\azcopy_windows_amd64.exe" --version') do ( + # set raw_version_output=%%a %%b %%c + # ) + # for /f "tokens=3 delims= " %%a in ("%raw_version_output%") do ( + # set azcopy_version=%%a + # ) + # echo AzCopy Version: %azcopy_version% + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(windows_386)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # dir $(work_dir) + # dir $(windows_386) + # dir $(windows_amd64) + # displayName: 'List the binaries' + + # - script: | + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_386) + # displayName: 'Copy Notice file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64*.zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386*.zip' + # continueOnError: true + + # - script: | + # copy "%archives%\azcopy_windows_amd64*.zip" "%Build.ArtifactStagingDirectory%" + # copy "%archives%\azcopy_windows_386*.zip" "%Build.ArtifactStagingDirectory%" + # copy %work_dir%/azcopy_windows_v7_arm.exe "%Build.ArtifactStagingDirectory% + # displayName: 'Copy zip to staging directory' # - task: PublishBuildArtifacts@1 # inputs: @@ -429,112 +668,93 @@ stages: # verbosity: 'Verbose' # alertWarningLevel: 'High' - # - job: Set_2 + # - job: Set_4 # timeoutInMinutes: 120 # strategy: # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: 'blobfuse-ubn22-arm64' - # tags: 'tags' - + # MacOS: + # imageName: 'macos-11' + # type: 'mac-os' # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + # vmImage: $(imageName) # variables: # - name: root_dir # value: '$(System.DefaultWorkingDirectory)' # - name: work_dir # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: darwin_amd_dir + # value: '$(work_dir)/azcopy_darwin_amd64' + # - name: darwin_arm_dir + # value: '$(work_dir)/azcopy_darwin_arm64' + # - name: archives + # value: $(root_dir)/archives # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # version: $(AZCOPY_GOLANG_VERSION) # - script: | # echo 'Running GO Vet' # go vet - # displayName: 'Golang Vet - Linux' + # displayName: 'Golang Vet - Mac' # workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy # - script: | - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # mkdir $(darwin_amd_dir) + # mkdir $(darwin_arm_dir) + # mkdir $(archives) + # displayName: 'Create required directories' - # # get glibc version with which build is done # - script: | - # ldd --version - # displayName: "GLIBC Version" + # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + # raw_version_output=$("$(darwin_amd_dir)/azcopy_darwin_amd64" --version) + # IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + # for element in "${raw_version_output_array[@]}"; do + # echo "Array Element: $element" + # done - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirArm64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - # displayName: 'Accumulate arm64 pkg files' + # echo "Array Length: ${#raw_version_output_array[@]}" + # azcopy_version=${raw_version_output_array[2]} + # echo "AzCopy Version: $azcopy_version" + # displayName: 'Generate MacOS Build with AMD64' # - script: | - # cp NOTICE.txt pkgDirArm64/usr/bin/ - # displayName: 'Copy NOTICE.txt' + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_arm64" + # displayName: 'Generate MacOS Build with Arm64' # - script: | - # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - # displayName: 'Verify the generated build' + # sudo ls -lRt $(work_dir) + # displayName: 'List the amd64 files' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - # echo "Version : $version" - # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' - + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # displayName: 'Copy Notice file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$azcopy_version.zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$azcopy_version.zip' + # continueOnError: true + # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + # cp $(archives)/azcopy_darwin_amd64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_darwin_arm64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' # - task: PublishBuildArtifacts@1 # inputs: @@ -547,29 +767,19 @@ stages: # scanType: 'Register' # verbosity: 'Verbose' # alertWarningLevel: 'High' - - - job: Set_3 - timeoutInMinutes: 120 + + - job: Build + timeoutInMinutes: 360 strategy: matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' Windows: imageName: 'windows-2019' type: 'windows' pool: vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - - name: windows_amd64 - value: '$(work_dir)\azcopy_windows_amd64' - - name: windows_386 - value: '$(work_dir)\azcopy_windows_386' - - name: archives - value: $(root_dir)\archives - steps: - task: GoTool@0 env: @@ -580,41 +790,31 @@ stages: - script: | echo 'Running GO Vet' go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - mkdir $(windows_amd64) - mkdir $(windows_386) - mkdir $(archives) - displayName: 'Create required directories' + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) - script: | - go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - for /f "tokens=1-3 delims= " %%a in ('"$(windows_amd64)\azcopy_windows_amd64.exe" --version') do ( - set raw_version_output=%%a %%b %%c - ) - for /f "tokens=3 delims= " %%a in ("%raw_version_output%") do ( - set azcopy_version=%%a - ) - echo AzCopy Version: %azcopy_version% + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" displayName: 'Generate Windows AMD64' + condition: eq(variables.type, 'windows') env: GOARCH: amd64 GOOS: windows CGO_ENABLED: 0 - script: | - go build -o "$(windows_386)\azcopy_windows_386.exe" + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" displayName: 'Generate Windows i386' + condition: eq(variables.type, 'windows') env: GOARCH: 386 GOOS: windows CGO_ENABLED: 0 - script: | - go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" displayName: 'Generate Windows ARM' + condition: eq(variables.type, 'windows') env: GOARCH: arm GOARM: 7 @@ -622,57 +822,43 @@ stages: CGO_ENABLED: 0 - script: | - dir $(work_dir) - dir $(windows_386) - dir $(windows_amd64) - displayName: 'List the binaries' - - - script: | - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_386) - displayName: 'Copy Notice file' - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64*.zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386*.zip' - continueOnError: true + CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + condition: eq(variables.type, 'mac-os') - script: | - copy "%archives%\azcopy_windows_amd64*.zip" "%Build.ArtifactStagingDirectory%" - copy "%archives%\azcopy_windows_386*.zip" "%Build.ArtifactStagingDirectory%" - copy %work_dir%/azcopy_windows_v7_arm.exe "%Build.ArtifactStagingDirectory% - displayName: 'Copy zip to staging directory' + GOARCH=arm64 CGO_ENABLED=1 go build + displayName: 'Test Cross-compiled MacOS Build with ARM64' + condition: eq(variables.type, 'mac-os') - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() - + - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' + # BuildArtifacts end here - - job: Set_4 + - stage: CreateZip + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 timeoutInMinutes: 120 strategy: matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(imageName) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) variables: - name: root_dir @@ -685,19 +871,15 @@ stages: value: '$(work_dir)/azcopy_darwin_arm64' - name: archives value: $(root_dir)/archives - + steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(root_dir) + artifactName: 'azCopy-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-signed/azcopy_darwin* - script: | mkdir $(darwin_amd_dir) @@ -706,50 +888,29 @@ stages: displayName: 'Create required directories' - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - - raw_version_output=$("$(darwin_amd_dir)/azcopy_darwin_amd64" --version) - IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" - - for element in "${raw_version_output_array[@]}"; do - echo "Array Element: $element" - done - - echo "Array Length: ${#raw_version_output_array[@]}" - azcopy_version=${raw_version_output_array[2]} - echo "AzCopy Version: $azcopy_version" - displayName: 'Generate MacOS Build with AMD64' - - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_arm64" - displayName: 'Generate MacOS Build with Arm64' - - - script: | - sudo ls -lRt $(work_dir) - displayName: 'List the amd64 files' - - - script: | + cp $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 $(darwin_amd_dir) cp NOTICE.txt $(darwin_amd_dir) + cp $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 $(darwin_arm_dir) cp NOTICE.txt $(darwin_arm_dir) - displayName: 'Copy Notice file' + displayName: 'Copy required file' - task: ArchiveFiles@2 displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$azcopy_version.zip' + archiveFile: '$(archives)/azcopy_darwin_amd64.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$azcopy_version.zip' + archiveFile: '$(archives)/azcopy_darwin_arm64.zip' continueOnError: true - script: | - cp $(archives)/azcopy_darwin_amd64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_amd64.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64.zip $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 @@ -763,12 +924,10 @@ stages: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' - -# BuildArtifacts end here - + - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') + dependsOn: CreateZip + condition: succeeded('CreateZip') jobs: - job: Set_1 timeoutInMinutes: 120 From 0c4ef97f69234dc8f10b5af16f086881d102bad7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 14:23:37 +0530 Subject: [PATCH 142/513] test windows and mac buildworkflow --- release-pipelines.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index aa8914ef0..e2017572c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -821,6 +821,18 @@ stages: GOOS: windows CGO_ENABLED: 0 + - script: | + for /f "tokens=*" %%i in ('$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe --version') do set "azcopy_output=%%i" + set "full_version=!azcopy_output:azcopy version =!" + set "position=0" + set "temp_string=!full_version!" + :find_dash + if not "!temp_string:~%position%,1!"=="-" ( + set /a position+=1 + goto find_dash + ) + set "version=!full_version:~0,%position%!" + echo Version: !version! - script: | CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" displayName: 'Generate MacOS Build with AMD64' @@ -879,7 +891,7 @@ stages: inputs: artifactName: 'azCopy-temp' downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-signed/azcopy_darwin* + itemPattern: azCopy-temp/azcopy_darwin* - script: | mkdir $(darwin_amd_dir) From 66f0cee72270dc04b5cfe9209cc96a0f97b1a49f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 14:40:30 +0530 Subject: [PATCH 143/513] test windows and mac buildworkflow --- release-pipelines.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index e2017572c..74aa449fd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -833,6 +833,9 @@ stages: ) set "version=!full_version:~0,%position%!" echo Version: !version! + displayName: 'Get version' + condition: eq(variables.type, 'windows') + - script: | CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" displayName: 'Generate MacOS Build with AMD64' @@ -843,6 +846,13 @@ stages: displayName: 'Test Cross-compiled MacOS Build with ARM64' condition: eq(variables.type, 'mac-os') + - script: | + azcopy_output=$($(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 --version) + version=$(echo "$azcopy_output" | awk '{print $3}' | cut -d'-' -f1) + echo "Version: $version" + displayName: 'Get version' + condition: eq(variables.type, 'mac-os') + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' From e42335bfa5de1d34024cd708512071e9dabbbe9e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:02:24 +0530 Subject: [PATCH 144/513] test windows and mac buildworkflow --- release-pipelines.yml | 403 ++++++++++++------------------------------ 1 file changed, 117 insertions(+), 286 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 74aa449fd..8bfd12aa8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -552,234 +552,28 @@ stages: # verbosity: 'Verbose' # alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - # - name: windows_amd64 - # value: '$(work_dir)\azcopy_windows_amd64' - # - name: windows_386 - # value: '$(work_dir)\azcopy_windows_386' - # - name: archives - # value: $(root_dir)\archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # mkdir $(windows_amd64) - # mkdir $(windows_386) - # mkdir $(archives) - # displayName: 'Create required directories' - - # - script: | - # go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - # for /f "tokens=1-3 delims= " %%a in ('"$(windows_amd64)\azcopy_windows_amd64.exe" --version') do ( - # set raw_version_output=%%a %%b %%c - # ) - # for /f "tokens=3 delims= " %%a in ("%raw_version_output%") do ( - # set azcopy_version=%%a - # ) - # echo AzCopy Version: %azcopy_version% - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(windows_386)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # dir $(work_dir) - # dir $(windows_386) - # dir $(windows_amd64) - # displayName: 'List the binaries' - - # - script: | - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_386) - # displayName: 'Copy Notice file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64*.zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386*.zip' - # continueOnError: true - - # - script: | - # copy "%archives%\azcopy_windows_amd64*.zip" "%Build.ArtifactStagingDirectory%" - # copy "%archives%\azcopy_windows_386*.zip" "%Build.ArtifactStagingDirectory%" - # copy %work_dir%/azcopy_windows_v7_arm.exe "%Build.ArtifactStagingDirectory% - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() - - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-11' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: darwin_amd_dir - # value: '$(work_dir)/azcopy_darwin_amd64' - # - name: darwin_arm_dir - # value: '$(work_dir)/azcopy_darwin_arm64' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(root_dir) - - # - script: | - # mkdir $(darwin_amd_dir) - # mkdir $(darwin_arm_dir) - # mkdir $(archives) - # displayName: 'Create required directories' - - # - script: | - # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - - # raw_version_output=$("$(darwin_amd_dir)/azcopy_darwin_amd64" --version) - # IFS=$'\n' read -rd '' -a raw_version_output_array <<<"$raw_version_output" - - # for element in "${raw_version_output_array[@]}"; do - # echo "Array Element: $element" - # done - - # echo "Array Length: ${#raw_version_output_array[@]}" - # azcopy_version=${raw_version_output_array[2]} - # echo "AzCopy Version: $azcopy_version" - # displayName: 'Generate MacOS Build with AMD64' - - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(work_dir)/azcopy_darwin_arm64" - # displayName: 'Generate MacOS Build with Arm64' - - # - script: | - # sudo ls -lRt $(work_dir) - # displayName: 'List the amd64 files' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # displayName: 'Copy Notice file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$azcopy_version.zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$azcopy_version.zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$azcopy_version.zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() - - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - - job: Build - timeoutInMinutes: 360 + - job: Set_3 + timeoutInMinutes: 120 strategy: matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' Windows: imageName: 'windows-2019' type: 'windows' pool: vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)\azcopy_windows_amd64' + - name: windows_386 + value: '$(work_dir)\azcopy_windows_386' + - name: archives + value: $(root_dir)\archives + steps: - task: GoTool@0 env: @@ -790,97 +584,115 @@ stages: - script: | echo 'Running GO Vet' go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe" + mkdir $(windows_amd64) + mkdir $(windows_386) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" displayName: 'Generate Windows AMD64' - condition: eq(variables.type, 'windows') env: GOARCH: amd64 GOOS: windows CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe" + go build -o "$(windows_386)\azcopy_windows_386.exe" displayName: 'Generate Windows i386' - condition: eq(variables.type, 'windows') env: GOARCH: 386 GOOS: windows CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe" + go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" displayName: 'Generate Windows ARM' - condition: eq(variables.type, 'windows') env: GOARCH: arm GOARM: 7 GOOS: windows CGO_ENABLED: 0 - - script: | - for /f "tokens=*" %%i in ('$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe --version') do set "azcopy_output=%%i" - set "full_version=!azcopy_output:azcopy version =!" - set "position=0" - set "temp_string=!full_version!" - :find_dash - if not "!temp_string:~%position%,1!"=="-" ( - set /a position+=1 - goto find_dash - ) - set "version=!full_version:~0,%position%!" - echo Version: !version! - displayName: 'Get version' - condition: eq(variables.type, 'windows') - - - script: | - CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' - condition: eq(variables.type, 'mac-os') + - script: | + dir $(work_dir) + dir $(windows_386) + dir $(windows_amd64) + displayName: 'List the binaries' - script: | - GOARCH=arm64 CGO_ENABLED=1 go build - displayName: 'Test Cross-compiled MacOS Build with ARM64' - condition: eq(variables.type, 'mac-os') + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_386) + displayName: 'Copy Notice file' - - script: | - azcopy_output=$($(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 --version) - version=$(echo "$azcopy_output" | awk '{print $3}' | cut -d'-' -f1) - echo "Version: $version" - displayName: 'Get version' - condition: eq(variables.type, 'mac-os') + # Add your other pipeline steps here + + - task: PowerShell@2 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + # Run azcopy --version and capture the output + $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.ex --version + + # Example output: "azcopy version 10.25.0-Preview-1" + # Extract the version part using string manipulation + if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { + $version = $matches[1] + Write-Output "Extracted version: $version" + + # Set the pipeline variable + Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + } else { + Write-Error "Unable to extract version from azcopy output" + } + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + copy "%archives%\azcopy_windows_amd64_$(azcopy_version).zip" "%Build.ArtifactStagingDirectory%" + copy "%archives%\azcopy_windows_386_$(azcopy_version).zip" "%Build.ArtifactStagingDirectory%" + copy %work_dir%/azcopy_windows_v7_arm.exe "%Build.ArtifactStagingDirectory% + displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() - + - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' - # BuildArtifacts end here - - stage: CreateZip - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 + + - job: Set_4 + timeoutInMinutes: 360 strategy: matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" + MacOS: + imageName: 'macos-11' + type: 'mac-os' pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + vmImage: $(imageName) variables: - name: root_dir @@ -893,15 +705,19 @@ stages: value: '$(work_dir)/azcopy_darwin_arm64' - name: archives value: $(root_dir)/archives - + steps: - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' + - task: GoTool@0 + env: + GO111MODULE: 'on' inputs: - artifactName: 'azCopy-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-temp/azcopy_darwin* + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) - script: | mkdir $(darwin_amd_dir) @@ -910,29 +726,43 @@ stages: displayName: 'Create required directories' - script: | - cp $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 $(darwin_amd_dir) + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build $(darwin_arm_dir)/azcopy_darwin_arm64 + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | cp NOTICE.txt $(darwin_amd_dir) - cp $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 $(darwin_arm_dir) cp NOTICE.txt $(darwin_arm_dir) - displayName: 'Copy required file' + displayName: 'Copy NOTICE file' - task: ArchiveFiles@2 displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64.zip' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' inputs: rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64.zip' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' continueOnError: true - script: | - cp $(archives)/azcopy_darwin_amd64.zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 @@ -946,7 +776,8 @@ stages: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' - + # BuildArtifacts end here + - stage: SignArtifacts dependsOn: CreateZip condition: succeeded('CreateZip') From fcf5bc2ec6aa47fbfed41035995948cc65aa9747 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:03:20 +0530 Subject: [PATCH 145/513] test windows and mac buildworkflow --- release-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8bfd12aa8..be8a3bfb9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -735,11 +735,11 @@ stages: - task: Bash@3 displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - script: | cp NOTICE.txt $(darwin_amd_dir) From c84ed3fdf9134182bc8c33930b831da467210176 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:04:10 +0530 Subject: [PATCH 146/513] test windows and mac buildworkflow --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index be8a3bfb9..23944b12e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -779,8 +779,8 @@ stages: # BuildArtifacts end here - stage: SignArtifacts - dependsOn: CreateZip - condition: succeeded('CreateZip') + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') jobs: - job: Set_1 timeoutInMinutes: 120 From 326205497476a2c90e14168f1231f868cf0789e1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:14:12 +0530 Subject: [PATCH 147/513] test windows and mac buildworkflow --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 23944b12e..08ebd9a0c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -637,7 +637,7 @@ stages: targetType: 'inline' script: | # Run azcopy --version and capture the output - $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.ex --version + $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version # Example output: "azcopy version 10.25.0-Preview-1" # Extract the version part using string manipulation @@ -730,7 +730,7 @@ stages: displayName: 'Generate MacOS Build with AMD64' - script: | - GOARCH=arm64 CGO_ENABLED=1 go build $(darwin_arm_dir)/azcopy_darwin_arm64 + GOARCH=arm64 CGO_ENABLED=1 go build "$(darwin_arm_dir)/azcopy_darwin_arm64" displayName: 'Test Cross-compiled MacOS Build with ARM64' - task: Bash@3 From 99c70ea9a1d05505da95913d6691a7e148b0d424 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:19:20 +0530 Subject: [PATCH 148/513] test windows and mac buildworkflow --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 08ebd9a0c..39c5b3485 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -730,7 +730,7 @@ stages: displayName: 'Generate MacOS Build with AMD64' - script: | - GOARCH=arm64 CGO_ENABLED=1 go build "$(darwin_arm_dir)/azcopy_darwin_arm64" + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" displayName: 'Test Cross-compiled MacOS Build with ARM64' - task: Bash@3 From 6260674831cb56c56033fd299bde5a3ac186a5d0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:25:12 +0530 Subject: [PATCH 149/513] test windows and mac buildworkflow --- release-pipelines.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 39c5b3485..4889ff79e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -665,11 +665,14 @@ stages: archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' continueOnError: true - - script: | - copy "%archives%\azcopy_windows_amd64_$(azcopy_version).zip" "%Build.ArtifactStagingDirectory%" - copy "%archives%\azcopy_windows_386_$(azcopy_version).zip" "%Build.ArtifactStagingDirectory%" - copy %work_dir%/azcopy_windows_v7_arm.exe "%Build.ArtifactStagingDirectory% + - task: PowerShell@2 displayName: 'Copy zip to staging directory' + inputs: + targetType: 'inline' + script: | + Copy-Item -Path "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" + Copy-Item -Path "$(archives)\azcopy_windows_386_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" + Copy-Item -Path "$(work_dir)\azcopy_windows_v7_arm.exe" -Destination "$(Build.ArtifactStagingDirectory)" - task: PublishBuildArtifacts@1 inputs: From 0afe3a23a93a092cea6dd6db7b2de9ea897311e0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 15:42:58 +0530 Subject: [PATCH 150/513] test windows and mac buildworkflow --- release-pipelines.yml | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4889ff79e..252943b7a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -670,9 +670,46 @@ stages: inputs: targetType: 'inline' script: | - Copy-Item -Path "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" - Copy-Item -Path "$(archives)\azcopy_windows_386_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" - Copy-Item -Path "$(work_dir)\azcopy_windows_v7_arm.exe" -Destination "$(Build.ArtifactStagingDirectory)" + # Ensure the destination directory exists + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force + + # Copy the zip files and executable to the artifact staging directory + $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" + $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" + $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" + $destinationDir = "$(Build.ArtifactStagingDirectory)" + + # Check if the source files exist before copying + if (Test-Path $sourceAmd64Zip) { + Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceAmd64Zip to $destinationDir" + } else { + Write-Error "File not found: $sourceAmd64Zip" + } + + if (Test-Path $source386Zip) { + Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $source386Zip to $destinationDir" + } else { + Write-Error "File not found: $source386Zip" + } + + if (Test-Path $sourceArmExe) { + Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceArmExe to $destinationDir" + } else { + Write-Error "File not found: $sourceArmExe" + } + + + # - task: PowerShell@2 + # displayName: 'Copy zip to staging directory' + # inputs: + # targetType: 'inline' + # script: | + # Copy-Item -Path "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" + # Copy-Item -Path "$(archives)\azcopy_windows_386_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" + # Copy-Item -Path "$(work_dir)\azcopy_windows_v7_arm.exe" -Destination "$(Build.ArtifactStagingDirectory)" - task: PublishBuildArtifacts@1 inputs: From 017e65fbfc7b093e7a83f8044f942895ce29755c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 16:27:39 +0530 Subject: [PATCH 151/513] test windows and mac buildworkflow --- release-pipelines.yml | 212 +++++++++++++++++++++--------------------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 252943b7a..a718fdc6b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -822,120 +822,120 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + # displayName: 'List Artifacts' - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-temp/*arm64.rpm mariner - cp azCopy-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-temp/*arm64.rpm mariner + # cp azCopy-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get update + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ - displayName: 'Make Artifacts executable' + # # Send images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + # displayName: 'Make Artifacts executable' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-signed' + # displayName: 'Publish Signed Artifacts' - - job: Set_2 + - job: Set_1 timeoutInMinutes: 120 strategy: matrix: @@ -1030,8 +1030,8 @@ stages: displayName: 'ESRP CodeSigning azCopy windows' inputs: ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/windows' - Pattern: '*.exe' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*.zip' signConfigType: inlineSignParams VerboseLogin: true inlineOperation: | @@ -1071,7 +1071,7 @@ stages: displayName: 'ESRP CodeSigning azCopy macos' inputs: ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mac' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' Pattern: 'azcopy_darwin*' signConfigType: inlineSignParams VerboseLogin: true From 917a73147e471b834cce36e57d33902114fbed4e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 16:59:15 +0530 Subject: [PATCH 152/513] test windows and mac buildworkflow --- release-pipelines.yml | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a718fdc6b..ba888855a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1030,43 +1030,20 @@ stages: displayName: 'ESRP CodeSigning azCopy windows' inputs: ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: '*.zip' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' signConfigType: inlineSignParams VerboseLogin: true inlineOperation: | [ { - "KeyCode" : "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "OperationSetCode": "SigntoolSign", - "Parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "ToolName": "signtool.exe", - "ToolVersion": "6.2.9304.0" - + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" } ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: 'ESRP CodeSigning azCopy macos' inputs: From d76e1db54c102b3b9ae37d1ba9f85bbc2119d577 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 16:59:48 +0530 Subject: [PATCH 153/513] test windows and mac buildworkflow --- release-pipelines.yml | 44 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ba888855a..c51b20c74 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1030,40 +1030,38 @@ stages: displayName: 'ESRP CodeSigning azCopy windows' inputs: ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*.zip, *.exe' signConfigType: inlineSignParams VerboseLogin: true inlineOperation: | [ { "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", + "OperationCode" : "WindowsSign", "Parameters" : {}, "ToolName" : "sign", "ToolVersion" : "1.0" } ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy macos' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning macOS AzCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", + "OperationCode" : "MacOSSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] # Validate signed images have md5sum changed - script: | From 8d0620e63412e2872cf1318227bfcb6b65eaee07 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 6 Jun 2024 17:01:13 +0530 Subject: [PATCH 154/513] test windows and mac buildworkflow --- release-pipelines.yml | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c51b20c74..a700937fc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1044,24 +1044,25 @@ stages: "ToolVersion" : "1.0" } ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning macOS AzCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", - "OperationCode" : "MacOSSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning macOS AzCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", + "OperationCode" : "MacOSSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] # Validate signed images have md5sum changed - script: | From 3d66b432d47870e6b6041dabbeef1c4dcd917ac4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 11:53:16 +0530 Subject: [PATCH 155/513] test complete workflow --- release-pipelines.yml | 583 +++++++++++++++++++----------------------- 1 file changed, 261 insertions(+), 322 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a700937fc..81e51c0e4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -311,7 +311,6 @@ stages: matrix: Ubuntu_amd64: imageName: 'ubuntu-20.04' - tags: 'tags' container: 'test-cnt-ubn-20' AgentName: "blobfuse-ubuntu20" pool: @@ -393,14 +392,13 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.deb + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' @@ -410,8 +408,8 @@ stages: -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "Add description" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`-$(tags).x86_64.rpm + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -433,124 +431,123 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: 'blobfuse-ubn22-arm64' - # tags: 'tags' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' - - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) - - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirArm64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - # displayName: 'Accumulate arm64 pkg files' - - # - script: | - # cp NOTICE.txt pkgDirArm64/usr/bin/ - # displayName: 'Copy NOTICE.txt' - - # - script: | - # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - # displayName: 'Verify the generated build' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - # echo "Version : $version" - # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: 'blobfuse-ubn22-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirArm64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + displayName: 'Accumulate arm64 pkg files' + + - script: | + cp NOTICE.txt pkgDirArm64/usr/bin/ + displayName: 'Copy NOTICE.txt' + + - script: | + GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + displayName: 'Verify the generated build' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + echo "Version : $version" + echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "Add description" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`-$(tags).arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -629,8 +626,6 @@ stages: cp NOTICE.txt $(windows_386) displayName: 'Copy Notice file' - # Add your other pipeline steps here - - task: PowerShell@2 displayName: 'Extract AZCopy version' inputs: @@ -701,16 +696,6 @@ stages: Write-Error "File not found: $sourceArmExe" } - - # - task: PowerShell@2 - # displayName: 'Copy zip to staging directory' - # inputs: - # targetType: 'inline' - # script: | - # Copy-Item -Path "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" - # Copy-Item -Path "$(archives)\azcopy_windows_386_$(azcopy_version).zip" -Destination "$(Build.ArtifactStagingDirectory)" - # Copy-Item -Path "$(work_dir)\azcopy_windows_v7_arm.exe" -Destination "$(Build.ArtifactStagingDirectory)" - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-temp' @@ -822,118 +807,118 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-temp/*arm64.rpm mariner - # cp azCopy-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-temp/*arm64.rpm mariner + cp azCopy-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get update - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # Send images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-signed' + displayName: 'Publish Signed Artifacts' - job: Set_1 timeoutInMinutes: 120 @@ -989,7 +974,7 @@ stages: # Send images for signing - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy' + displayName: 'ESRP CodeSigning azCopy linux' inputs: ConnectedServiceName: 'PMC ESRP AzCopy Signing' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' @@ -1100,7 +1085,6 @@ stages: agentName: "blobfuse-ubuntu22" vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22' - tags: 'tags' pool: vmImage: $(vmImage) @@ -1137,7 +1121,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*$(tags)*x86_64.deb + itemPattern: azCopy-signed/azcopy-*x86_64.deb - script: | ls -l @@ -1146,12 +1130,6 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed - - - script: | - for f in ./azcopy*$(tags)*.x86_64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*x86_64.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' workingDirectory: $(root_dir)/azCopy-signed - script: | @@ -1169,12 +1147,6 @@ stages: azcopy_linux_amd64 --help displayName: 'Check Help' - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy' - displayName: 'Publish Artifacts' - - job: Set_2 timeoutInMinutes: 120 strategy: @@ -1183,7 +1155,6 @@ stages: vmImage: 'Ubuntu-22.04' container: 'test-cnt-ubn-22-arm64' AgentName: "blobfuse-ubn22-arm64" - tags: 'tags' pool: name: "blobfuse-ubn-arm64-pool" demands: @@ -1228,7 +1199,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*$(tags)*.arm64.deb + itemPattern: azCopy-signed/azcopy-*.arm64.deb - script: | ls -l @@ -1239,12 +1210,6 @@ stages: displayName: 'List Downloaded Package' workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy*$(tags)*.arm64.deb; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.arm64.deb $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed - - script: | sudo dpkg --info azcopy*.arm64.deb sudo dpkg -i azcopy*.arm64.deb @@ -1259,12 +1224,6 @@ stages: azcopy_linux_arm64 --help displayName: 'Check Help' - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy' - displayName: 'Publish Artifacts' - - job: Set_3 timeoutInMinutes: 120 strategy: @@ -1272,7 +1231,6 @@ stages: Rhel_Arm64: agentName: "blobfuse-rhel9-arm64" vmImage: 'RHEL-9.0' - tags: 'tags' container: 'test-cnt-rhel-9-arm64' pool: @@ -1322,7 +1280,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*$(tags)*arm64.rpm + itemPattern: azCopy-signed/azcopy-*arm64.rpm - script: | ls -l @@ -1333,17 +1291,11 @@ stages: displayName: 'List Downloaded Package' workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed - - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy*$(vmImage)*.rpm + sudo rpm -qip azcopy-*arm64.rpm sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy*$(vmImage)*.rpm + sudo rpm -i azcopy-*arm64.rpm displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1354,12 +1306,6 @@ stages: - script: | azcopy_linux_arm64 --help displayName: 'Check Help' - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy' - displayName: 'Publish Artifacts' - job: Set_4 timeoutInMinutes: 120 @@ -1368,7 +1314,6 @@ stages: Rhel_Amd64: agentName: "blobfuse-rhel9" vmImage: 'RHEL-9.0' - tags: 'tags' container: 'test-cnt-rhel-9' pool: @@ -1418,7 +1363,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy*$(tags)*x86_64.rpm + itemPattern: azCopy-signed/azcopy*x86_64.rpm - script: | ls -l @@ -1429,17 +1374,11 @@ stages: displayName: 'List Downloaded Package' workingDirectory: $(root_dir)/azCopy-signed - - script: | - for f in ./azcopy*$(tags)*.rpm; do mv -v "$f" "${f/-$(tags)./-$(vmImage).}"; done; - cp ./azcopy*$(vmImage)*.rpm $(Build.ArtifactStagingDirectory) - displayName: 'Rename Package' - workingDirectory: $(root_dir)/azCopy-signed - - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy*$(vmImage)*.rpm + sudo rpm -qip azcopy*x86_64.rpm sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy*$(vmImage)*.rpm + sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1451,12 +1390,6 @@ stages: azcopy_linux_amd64 --help displayName: 'Check Help' - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy' - displayName: 'Publish Artifacts' - - job: Set_5 timeoutInMinutes: 120 strategy: @@ -1466,7 +1399,6 @@ stages: DistroVer: "Mariner2" Description: "CBL-Mariner2 Linux" container: "test-cnt-mari-2" - tags: 'tags' pool: name: "blobfuse-mariner-pool" @@ -1515,7 +1447,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/mariner/azcopy*$(tags)*x86_64.rpm + itemPattern: azCopy-signed/mariner/azcopy*x86_64.rpm - script: | ls -l @@ -1558,9 +1490,6 @@ stages: demands: - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET - steps: - checkout: none @@ -1572,7 +1501,7 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy' + artifactName: 'azCopy-signed' downloadPath: $(Build.ArtifactStagingDirectory) - script: | @@ -1592,6 +1521,7 @@ stages: - ${{ if eq(parameters.post_release, true) }}: # add release tags & push to github + #TODO: update github connection and repo name while merging to main - task: GithubRelease@1 inputs: githubConnection: 'github.com_dphulkar-msft' @@ -1605,7 +1535,7 @@ stages: tag: ${{ parameters.tag }} assets: | - $(Build.ArtifactStagingDirectory)/azCopy/* + $(Build.ArtifactStagingDirectory)/azCopy-signed/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} @@ -1665,28 +1595,31 @@ stages: artifactName: 'azCopy-signed' downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - cd mariner - for f in ./azcopy*.rpm; do mv -v "$f" "${f/-tags./-cm2.}"; done - ls -lRt - mv azcopy*.rpm .. - cd .. - rm -r mariner/ + - task: Bash@3 displayName: 'Rename Mariner binaries' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + inputs: + targetType: 'inline' + script: | + cd mariner + marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy-10.25.0-Preview-1-cm2.arm64.rpm" + marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy-10.25.0-Preview-1-cm2.x86_64.rpm" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName"; done + mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName"; done + ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) displayName: 'List Artifacts' - script: | - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-cm2.x86_64.rpm - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.x86_64.rpm - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.x86_64.deb - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-cm2.arm64.rpm - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.arm64.rpm - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azcopy-10.25.0-Preview-1-tags.arm64.deb - displayName: 'Upload files' + mv azCopy-signed/azcopy_linux_se_amd64 . + pmc package upload azCopy-signed + mv azcopy_linux_se_amd64 azCopy-signed + displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - script: | @@ -1696,6 +1629,12 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | + marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.arm64.rpm` + echo "Azcopy AMD DEB ID: $marinerArmRpm" + + marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` + echo "Azcopy AMD DEB ID: $marinerAmdRpm" + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" From 3de160681ae4e2c64bdc67a3905f8b7827a4b167 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 11:55:06 +0530 Subject: [PATCH 156/513] test complete workflow --- release-pipelines.yml | 187 +++++++++--------------------------------- 1 file changed, 37 insertions(+), 150 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 81e51c0e4..b4a5cfb1d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -807,119 +807,6 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-temp/*arm64.rpm mariner - cp azCopy-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get update - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-signed' - displayName: 'Publish Signed Artifacts' - - job: Set_1 timeoutInMinutes: 120 strategy: @@ -1011,43 +898,43 @@ stages: } ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy windows' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: '*.zip, *.exe' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "WindowsSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning macOS AzCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", - "OperationCode" : "MacOSSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy windows' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + # Pattern: '*.zip, *.exe' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "WindowsSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning macOS AzCopy' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", + # "OperationCode" : "MacOSSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] # Validate signed images have md5sum changed - script: | From 1aefadc966306166a4010873a565a986c4c14eef Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 11:56:27 +0530 Subject: [PATCH 157/513] test complete workflow --- release-pipelines.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b4a5cfb1d..6fc05eacc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1484,13 +1484,12 @@ stages: - task: Bash@3 displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ inputs: targetType: 'inline' script: | cd mariner - marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy-10.25.0-Preview-1-cm2.arm64.rpm" - marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy-10.25.0-Preview-1-cm2.x86_64.rpm" + marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.arm64.rpm" + marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.x86_64.rpm" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName"; done mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName"; done ls -lRt From 493cbf488a226d35e410ee9f5eeffc55c9d2a5d4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 12:25:35 +0530 Subject: [PATCH 158/513] test complete workflow --- release-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6fc05eacc..4bf1c30d8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1020,8 +1020,8 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy-*.x86_64.deb - sudo dpkg -i azcopy-*.x86_64.deb + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1086,7 +1086,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*.arm64.deb + itemPattern: azCopy-signed/azcopy-*arm64.deb - script: | ls -l @@ -1098,8 +1098,8 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy*.arm64.deb - sudo dpkg -i azcopy*.arm64.deb + sudo dpkg --info azcopy*arm64.deb + sudo dpkg -i azcopy*arm64.deb displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1346,8 +1346,8 @@ stages: workingDirectory: $(root_dir)/azCopy-signed/mariner - script: | - sudo rpm -qip azcopy*.rpm - sudo rpm -i azcopy*.rpm + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' workingDirectory: $(root_dir)/azCopy-signed/mariner From ee90df0e1b5f2ac228d3a7ffa21dfc805de5501b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 12:30:07 +0530 Subject: [PATCH 159/513] test complete workflow --- release-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4bf1c30d8..c99e24a84 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1098,8 +1098,8 @@ stages: workingDirectory: $(root_dir)/azCopy-signed - script: | - sudo dpkg --info azcopy*arm64.deb - sudo dpkg -i azcopy*arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1250,7 +1250,7 @@ stages: inputs: artifactName: 'azCopy-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy*x86_64.rpm + itemPattern: azCopy-signed/azcopy-*x86_64.rpm - script: | ls -l @@ -1263,9 +1263,9 @@ stages: - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -qip azcopy-*x86_64.rpm sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy*x86_64.rpm + sudo rpm -i azcopy-*x86_64.rpm displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) From d219b8b88063c2c7c038325510e6629e0c2c7953 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 12:32:08 +0530 Subject: [PATCH 160/513] test complete workflow --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c99e24a84..f3b4589b6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1180,9 +1180,9 @@ stages: - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*arm64.rpm + sudo rpm -qip azcopy-*.arm64.rpm sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*arm64.rpm + sudo rpm -i azcopy-*.arm64.rpm displayName: 'Install Package' workingDirectory: $(Build.ArtifactStagingDirectory) From e63cdaa4a15b83bb67d8b3a8def229226066b8ab Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 12:56:00 +0530 Subject: [PATCH 161/513] test complete workflow --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f3b4589b6..6896b2319 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1024,7 +1024,7 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir)/azCopy-signed - script: | azcopy_linux_amd64 --version @@ -1101,7 +1101,7 @@ stages: sudo dpkg --info azcopy-*arm64.deb sudo dpkg -i azcopy-*arm64.deb displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir)/azCopy-signed - script: | azcopy_linux_arm64 --version @@ -1184,7 +1184,7 @@ stages: sudo yum groupinstall "Development Tools" -y sudo rpm -i azcopy-*.arm64.rpm displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir)/azCopy-signed - script: | azcopy_linux_arm64 --version @@ -1267,7 +1267,7 @@ stages: sudo yum groupinstall "Development Tools" -y sudo rpm -i azcopy-*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir)/azCopy-signed - script: | azcopy_linux_amd64 --version From 416aab8a0af9a4cae30a8ce5f8e7778e22ba0c05 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 13:31:06 +0530 Subject: [PATCH 162/513] test complete workflow --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6896b2319..87ff8a557 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1393,8 +1393,8 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azCopy/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-signed/*.rpm displayName: 'List Artifacts' - script: | From 277d15020e263ed0bedfe828794e35d042e1077f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 20:14:31 +0530 Subject: [PATCH 163/513] test complete workflow --- release-pipelines.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 87ff8a557..c0d7afb88 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1482,20 +1482,18 @@ stages: artifactName: 'azCopy-signed' downloadPath: $(Build.ArtifactStagingDirectory) - - task: Bash@3 - displayName: 'Rename Mariner binaries' - inputs: - targetType: 'inline' - script: | + - script: | cd mariner marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.arm64.rpm" marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.x86_64.rpm" - mv -v ./azcopy*.arm64.rpm "$marinerArmFileName"; done - mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName"; done + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName" ls -lRt mv azcopy*.rpm .. cd .. rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) From ef9769b02a41b02d8373821ac32a522faa640590 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 7 Jun 2024 20:15:39 +0530 Subject: [PATCH 164/513] test complete workflow --- release-pipelines.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c0d7afb88..2c8c2864c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1482,18 +1482,18 @@ stages: artifactName: 'azCopy-signed' downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - cd mariner - marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.arm64.rpm" - marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.x86_64.rpm" - mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName" - ls -lRt - mv azcopy*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + - script: | + cd mariner + marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.arm64.rpm" + marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.x86_64.rpm" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName" + ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) From 26fafab8210e452ca19043f1190b980f19f18a20 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sat, 8 Jun 2024 19:22:18 +0530 Subject: [PATCH 165/513] test complete workflow --- release-pipelines.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2c8c2864c..847718564 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1484,10 +1484,12 @@ stages: - script: | cd mariner - marinerArmFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.arm64.rpm" - marinerAmdFileName="$(Build.ArtifactStagingDirectory)/azcopy*-cm2.x86_64.rpm" + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/-1/-1-cm2}" + marinerAmdFileName="${amd64file/-1/-1-cm2}" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - mv -v ./azcopy*.X86_64.rpm "$marinerAmdFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" ls -lRt mv azcopy*.rpm .. cd .. @@ -1500,9 +1502,13 @@ stages: displayName: 'List Artifacts' - script: | - mv azCopy-signed/azcopy_linux_se_amd64 . + mv azcopy_linux_se_amd64 . + mv azcopy*.zip . + mv azcopy*.exe . pmc package upload azCopy-signed mv azcopy_linux_se_amd64 azCopy-signed + mv azcopy*.zip azCopy-signed + mv azcopy*.exe azCopy-signed displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ From 517450ec3795a9426aea00abcb88e546a272e48a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sat, 8 Jun 2024 20:37:55 +0530 Subject: [PATCH 166/513] test complete workflow --- release-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 847718564..7b9bba7e3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1502,15 +1502,15 @@ stages: displayName: 'List Artifacts' - script: | - mv azcopy_linux_se_amd64 . - mv azcopy*.zip . - mv azcopy*.exe . + mv azCopy-signed/azcopy_linux_se_amd64 . + mv azCopy-signed/azcopy*.zip . + mv azCopy-signed/azcopy*.exe . pmc package upload azCopy-signed - mv azcopy_linux_se_amd64 azCopy-signed - mv azcopy*.zip azCopy-signed - mv azcopy*.exe azCopy-signed + mv azcopy_linux_se_amd64 azCopy-signed/ + mv azcopy*.zip azCopy-signed/ + mv azcopy*.exe azCopy-signed/ displayName: 'Upload files' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + workingDirectory: $(Build.ArtifactStagingDirectory) - script: | wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv From c5458064093504ee94ba3586ecda9d8982363b22 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 09:42:56 +0530 Subject: [PATCH 167/513] test complete workflow --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7b9bba7e3..881a248cc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1509,7 +1509,7 @@ stages: mv azcopy_linux_se_amd64 azCopy-signed/ mv azcopy*.zip azCopy-signed/ mv azcopy*.exe azCopy-signed/ - displayName: 'Upload files' + displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) - script: | From 4965b9ab329c67d8feb7dcd6d12ee230044477f9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 09:51:31 +0530 Subject: [PATCH 168/513] testing github release --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 881a248cc..66cd2d070 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1413,7 +1413,7 @@ stages: inputs: githubConnection: 'github.com_dphulkar-msft' repositoryName: 'dphulkar-msft/azure-storage-azcopy' - action: 'edit' + action: 'create' target: '$(Build.SourceVersion)' tagSource: 'userSpecifiedTag' From 34b216480b0d8453d223edf45f5504114b166287 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 10:10:08 +0530 Subject: [PATCH 169/513] seperated packages folder --- release-pipelines.yml | 297 +++++++++++++++++++++++++++++------------- 1 file changed, 209 insertions(+), 88 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 66cd2d070..fb885c519 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -421,7 +421,7 @@ stages: - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-temp' + artifactName: 'azCopy-linux-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -539,7 +539,7 @@ stages: - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-temp' + artifactName: 'azCopy-linux-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -698,7 +698,7 @@ stages: - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-temp' + artifactName: 'azCopy-windows-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -708,7 +708,6 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_4 timeoutInMinutes: 360 strategy: @@ -792,7 +791,7 @@ stages: - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-temp' + artifactName: 'azCopy-mac-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -833,18 +832,18 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-temp' + artifactName: 'azCopy-linux-temp' downloadPath: $(Build.ArtifactStagingDirectory) - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-temp/* + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* displayName: 'List Artifacts' - script: | mkdir mariner && chmod 755 mariner - cp azCopy-temp/*arm64.rpm mariner - cp azCopy-temp/*x86_64.rpm mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner sudo ls -lRt mariner md5sum mariner/* displayName: 'Copy artifacts for Mariner' @@ -864,7 +863,7 @@ stages: displayName: 'ESRP CodeSigning azCopy linux' inputs: ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' Pattern: '*.rpm, *.deb' signConfigType: inlineSignParams VerboseLogin: true @@ -898,52 +897,14 @@ stages: } ] - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy windows' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - # Pattern: '*.zip, *.exe' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "WindowsSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning macOS AzCopy' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-temp' - # Pattern: 'azcopy_darwin*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", - # "OperationCode" : "MacOSSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - # Validate signed images have md5sum changed - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-temp/ + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ displayName: 'Make Artifacts executable' - script: | @@ -955,7 +916,171 @@ stages: # Push signed images to artifact directory - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy windows' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + Pattern: '*.zip, *.exe' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "WindowsSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get update + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning macOS AzCopy' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", + "OperationCode" : "MacOSSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1006,9 +1131,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*x86_64.deb + itemPattern: azCopy-linux-signed/azcopy-*x86_64.deb - script: | ls -l @@ -1017,14 +1142,14 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | azcopy_linux_amd64 --version @@ -1084,9 +1209,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*arm64.deb + itemPattern: azCopy-linux-signed/azcopy-*arm64.deb - script: | ls -l @@ -1095,13 +1220,13 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | sudo dpkg --info azcopy-*arm64.deb sudo dpkg -i azcopy-*arm64.deb displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | azcopy_linux_arm64 --version @@ -1165,9 +1290,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*arm64.rpm + itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm - script: | ls -l @@ -1176,7 +1301,7 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo @@ -1184,7 +1309,7 @@ stages: sudo yum groupinstall "Development Tools" -y sudo rpm -i azcopy-*.arm64.rpm displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | azcopy_linux_arm64 --version @@ -1248,9 +1373,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/azcopy-*x86_64.rpm + itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm - script: | ls -l @@ -1259,7 +1384,7 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo @@ -1267,7 +1392,7 @@ stages: sudo yum groupinstall "Development Tools" -y sudo rpm -i azcopy-*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-signed + workingDirectory: $(root_dir)/azCopy-linux-signed - script: | azcopy_linux_amd64 --version @@ -1332,9 +1457,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(root_dir) - itemPattern: azCopy-signed/mariner/azcopy*x86_64.rpm + itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm - script: | ls -l @@ -1343,13 +1468,13 @@ stages: exit 1 fi displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-signed/mariner + workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - script: | sudo rpm -qip azcopy*x86_64.rpm sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-signed/mariner + workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - script: | azcopy_linux_amd64 --version @@ -1388,13 +1513,13 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(Build.ArtifactStagingDirectory) - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-signed/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azCopy-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm displayName: 'List Artifacts' - script: | @@ -1422,7 +1547,7 @@ stages: tag: ${{ parameters.tag }} assets: | - $(Build.ArtifactStagingDirectory)/azCopy-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} @@ -1479,7 +1604,7 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Signed Artifacts' inputs: - artifactName: 'azCopy-signed' + artifactName: 'azCopy-linux-signed' downloadPath: $(Build.ArtifactStagingDirectory) - script: | @@ -1495,20 +1620,16 @@ stages: cd .. rm -r mariner/ displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) displayName: 'List Artifacts' - script: | - mv azCopy-signed/azcopy_linux_se_amd64 . - mv azCopy-signed/azcopy*.zip . - mv azCopy-signed/azcopy*.exe . - pmc package upload azCopy-signed - mv azcopy_linux_se_amd64 azCopy-signed/ - mv azcopy*.zip azCopy-signed/ - mv azcopy*.exe azCopy-signed/ + mv azCopy-linux-signed/azcopy_linux_se_amd64 . + pmc package upload azCopy-linux-signed + mv azcopy_linux_se_amd64 azCopy-linux-signed/ displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1542,7 +1663,7 @@ stages: pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | while IFS=, read -r distro fuseArchType repoName releaseName; do @@ -1555,7 +1676,7 @@ stages: fi done < <(tail -n +3 ../packages.csv) displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-signed/ + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ # - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts From b22c5520c3b7d849f6172093e08e23055a86f021 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 10:11:48 +0530 Subject: [PATCH 170/513] seperated packages folder --- release-pipelines.yml | 76 +++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index fb885c519..86256aa0d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -810,7 +810,7 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Ubuntu-22: + Linux-Signing: vmImage: 'Ubuntu-20.04' AgentName: "blobfuse-ubuntu20" pool: @@ -923,7 +923,7 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Ubuntu-22: + Windows-Signing: vmImage: 'Ubuntu-20.04' AgentName: "blobfuse-ubuntu20" pool: @@ -963,24 +963,24 @@ stages: displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy windows' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - Pattern: '*.zip, *.exe' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "WindowsSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy windows' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + # Pattern: '*.zip, *.exe' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "WindowsSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] # Validate signed images have md5sum changed - script: | @@ -1045,24 +1045,24 @@ stages: displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning macOS AzCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", - "OperationCode" : "MacOSSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning macOS AzCopy' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", + # "OperationCode" : "MacOSSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] # Validate signed images have md5sum changed - script: | From 0300acca4ae5706df5a38ef971753573f58845ed Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 10:13:55 +0530 Subject: [PATCH 171/513] seperated packages folder --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 86256aa0d..0587cd979 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -850,7 +850,7 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory) - script: | - sudo apt-get update + sudo apt-get update --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update @@ -954,7 +954,7 @@ stages: displayName: 'List Artifacts' - script: | - sudo apt-get update + sudo apt-get update --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update @@ -1036,7 +1036,7 @@ stages: displayName: 'List Artifacts' - script: | - sudo apt-get update + sudo apt-get update --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update @@ -1523,7 +1523,7 @@ stages: displayName: 'List Artifacts' - script: | - sudo apt-get update + sudo apt-get update --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update From a10c8ecb33af19c7d04ecebe64d656c9371e085a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 11:48:14 +0530 Subject: [PATCH 172/513] seperated packages folder --- release-pipelines.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0587cd979..9620defbd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -850,10 +850,11 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory) - script: | - sudo apt-get update --fix-missing + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb - sudo apt update sudo apt install apt-transport-https -y sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" @@ -954,10 +955,11 @@ stages: displayName: 'List Artifacts' - script: | - sudo apt-get update --fix-missing + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb - sudo apt update sudo apt install apt-transport-https -y sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" @@ -1036,7 +1038,9 @@ stages: displayName: 'List Artifacts' - script: | - sudo apt-get update --fix-missing + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt update From e01c3b4b8a4eef7b4a10d695950765ebe3c1782a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 9 Jun 2024 12:30:01 +0530 Subject: [PATCH 173/513] seperated packages folder --- release-pipelines.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9620defbd..45d767efb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -347,6 +347,7 @@ stages: # install dependencies required for compiling azcopy - script: | + sudo apt-get clean sudo apt-get update --fix-missing sudo apt-get install pkg-config libsecret-1-dev -y sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y @@ -474,6 +475,7 @@ stages: # install dependencies required for compiling azcopy - script: | + sudo apt-get clean sudo apt-get update --fix-missing sudo apt-get install pkg-config libsecret-1-dev wget -y sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y @@ -850,6 +852,7 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory) - script: | + sudo apt-get clean sudo apt-get update sudo apt-get upgrade -y sudo apt-get install -f --fix-missing @@ -955,6 +958,7 @@ stages: displayName: 'List Artifacts' - script: | + sudo apt-get clean sudo apt-get update sudo apt-get upgrade -y sudo apt-get install -f --fix-missing @@ -1038,6 +1042,7 @@ stages: displayName: 'List Artifacts' - script: | + sudo apt-get clean sudo apt-get update sudo apt-get upgrade -y sudo apt-get install -f --fix-missing @@ -1527,6 +1532,7 @@ stages: displayName: 'List Artifacts' - script: | + sudo apt-get clean sudo apt-get update --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb From 7a23c4af6ed5ecec4cafe8b996ac773609f8ae8e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 12:20:16 +0530 Subject: [PATCH 174/513] test windows and mac signing --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 17e8e5d6e..77052d872 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1002,7 +1002,7 @@ stages: AuthCertName: 'azcopy-auth' AuthSignCertName: 'azcopy-sign-test' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - Pattern: '*.exe','*.zip' + Pattern: '*.exe, *.zip' signConfigType: inlineSignParams inlineOperation: | [ From ca5065c108037a48103f853af8fe28c8bddd3842 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 12:21:42 +0530 Subject: [PATCH 175/513] test windows and mac signing --- release-pipelines.yml | 82 +++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 77052d872..242b8dfbe 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1005,36 +1005,36 @@ stages: Pattern: '*.exe, *.zip' signConfigType: inlineSignParams inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] SessionTimeout: 30 @@ -1137,17 +1137,17 @@ stages: Pattern: 'azcopy_darwin*' signConfigType: inlineSignParams inlineOperation: | - [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] + [ + { + "KeyCode":"CP-401337-Apple", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] SessionTimeout: 30 # Validate signed images have md5sum changed From d73f8c5e195c8454f8128ba1409ae51befd81322 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 15:11:08 +0530 Subject: [PATCH 176/513] test mac signing --- release-pipelines.yml | 567 +++++++++++++++++++++--------------------- 1 file changed, 287 insertions(+), 280 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 242b8dfbe..d592b88fb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -437,9 +437,9 @@ stages: strategy: matrix: Ubuntu_arm64: - vmImage: 'ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: 'blobfuse-ubn22-arm64' + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' pool: name: "blobfuse-ubn-arm64-pool" @@ -543,6 +543,11 @@ stages: workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' + - script: | + cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add SE binary to signing stage' + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' @@ -555,164 +560,164 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - - name: windows_amd64 - value: '$(work_dir)\azcopy_windows_amd64' - - name: windows_386 - value: '$(work_dir)\azcopy_windows_386' - - name: archives - value: $(root_dir)\archives + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + # - name: windows_amd64 + # value: '$(work_dir)\azcopy_windows_amd64' + # - name: windows_386 + # value: '$(work_dir)\azcopy_windows_386' + # - name: archives + # value: $(root_dir)\archives - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - mkdir $(windows_amd64) - mkdir $(windows_386) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(windows_386)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - dir $(work_dir) - dir $(windows_386) - dir $(windows_amd64) - displayName: 'List the binaries' - - - script: | - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_386) - displayName: 'Copy Notice file' - - - task: PowerShell@2 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - # Run azcopy --version and capture the output - $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version - - # Example output: "azcopy version 10.25.0-Preview-1" - # Extract the version part using string manipulation - if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { - $version = $matches[1] - Write-Output "Extracted version: $version" + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # mkdir $(windows_amd64) + # mkdir $(windows_386) + # mkdir $(archives) + # displayName: 'Create required directories' + + # - script: | + # go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(windows_386)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # dir $(work_dir) + # dir $(windows_386) + # dir $(windows_amd64) + # displayName: 'List the binaries' + + # - script: | + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_386) + # displayName: 'Copy Notice file' + + # - task: PowerShell@2 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # # Run azcopy --version and capture the output + # $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version + + # # Example output: "azcopy version 10.25.0-Preview-1" + # # Extract the version part using string manipulation + # if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { + # $version = $matches[1] + # Write-Output "Extracted version: $version" - # Set the pipeline variable - Write-Host "##vso[task.setvariable variable=azcopy_version]$version" - } else { - Write-Error "Unable to extract version from azcopy output" - } - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - task: PowerShell@2 - displayName: 'Copy zip to staging directory' - inputs: - targetType: 'inline' - script: | - # Ensure the destination directory exists - New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force - - # Copy the zip files and executable to the artifact staging directory - $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" - $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" - $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" - $destinationDir = "$(Build.ArtifactStagingDirectory)" - - # Check if the source files exist before copying - if (Test-Path $sourceAmd64Zip) { - Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceAmd64Zip to $destinationDir" - } else { - Write-Error "File not found: $sourceAmd64Zip" - } - - if (Test-Path $source386Zip) { - Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $source386Zip to $destinationDir" - } else { - Write-Error "File not found: $source386Zip" - } - - if (Test-Path $sourceArmExe) { - Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceArmExe to $destinationDir" - } else { - Write-Error "File not found: $sourceArmExe" - } - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # # Set the pipeline variable + # Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + # } else { + # Write-Error "Unable to extract version from azcopy output" + # } + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - task: PowerShell@2 + # displayName: 'Copy zip to staging directory' + # inputs: + # targetType: 'inline' + # script: | + # # Ensure the destination directory exists + # New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force + + # # Copy the zip files and executable to the artifact staging directory + # $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" + # $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" + # $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" + # $destinationDir = "$(Build.ArtifactStagingDirectory)" + + # # Check if the source files exist before copying + # if (Test-Path $sourceAmd64Zip) { + # Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop + # Write-Output "Copied $sourceAmd64Zip to $destinationDir" + # } else { + # Write-Error "File not found: $sourceAmd64Zip" + # } + + # if (Test-Path $source386Zip) { + # Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop + # Write-Output "Copied $source386Zip to $destinationDir" + # } else { + # Write-Error "File not found: $source386Zip" + # } + + # if (Test-Path $sourceArmExe) { + # Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop + # Write-Output "Copied $sourceArmExe to $destinationDir" + # } else { + # Write-Error "File not found: $sourceArmExe" + # } + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_4 timeoutInMinutes: 360 @@ -927,135 +932,135 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy windows' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - # Pattern: '*.zip, *.exe' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "WindowsSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - Pattern: '*.exe, *.zip' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 - - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # # displayName: 'ESRP CodeSigning azCopy windows' + # # inputs: + # # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + # # Pattern: '*.zip, *.exe' + # # signConfigType: inlineSignParams + # # VerboseLogin: true + # # inlineOperation: | + # # [ + # # { + # # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # # "OperationCode" : "WindowsSign", + # # "Parameters" : {}, + # # "ToolName" : "sign", + # # "ToolVersion" : "1.0" + # # } + # # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Windows exectuable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + # Pattern: '*.exe, *.zip' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "keyCode": "CP-230012", + # "operationSetCode": "SigntoolSign", + # "parameters": [ + # { + # "parameterName": "OpusName", + # "parameterValue": "Microsoft" + # }, + # { + # "parameterName": "OpusInfo", + # "parameterValue": "http://www.microsoft.com" + # }, + # { + # "parameterName": "PageHash", + # "parameterValue": "/NPH" + # }, + # { + # "parameterName": "FileDigest", + # "parameterValue": "/fd sha256" + # }, + # { + # "parameterName": "TimeStamp", + # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + # } + # ], + # "toolName": "signtool.exe", + # "toolVersion": "6.2.9304.0" + # } + # ] + # SessionTimeout: 30 + + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' - job: Set_3 timeoutInMinutes: 120 @@ -1715,8 +1720,10 @@ stages: - script: | mv azCopy-linux-signed/azcopy_linux_se_amd64 . + mv azCopy-linux-signed/azcopy_linux_se_arm64 . pmc package upload azCopy-linux-signed mv azcopy_linux_se_amd64 azCopy-linux-signed/ + mv azcopy_linux_se_arm64 azCopy-linux-signed/ displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) From 931a7dac9d51b8ad24ba03c613ead1a230a34148 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 15:28:40 +0530 Subject: [PATCH 177/513] test mac signing --- release-pipelines.yml | 76 +++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d592b88fb..f1d4972c2 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1109,51 +1109,51 @@ stages: displayName: "Update dependencies" - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning macOS AzCopy' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - # Pattern: 'azcopy_darwin*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MACOS_KEY_CODE)", - # "OperationCode" : "MacOSSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning macOS AzCopy' inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' + ConnectedServiceName: 'PMC ESRP AzCopy Signing' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' Pattern: 'azcopy_darwin*' signConfigType: inlineSignParams + VerboseLogin: true inlineOperation: | [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } + { + "KeyCode" : "CP-401337-Apple", + "OperationCode" : "MacAppDeveloperSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } ] - SessionTimeout: 30 + + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Mac executable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "KeyCode":"CP-401337-Apple", + # "OperationCode":"MacAppDeveloperSign", + # "Parameters":{ + # "Hardening":"--options=runtime" + # }, + # "ToolName":"sign", + # "ToolVersion":"1.0" + # } + # ] + # SessionTimeout: 30 # Validate signed images have md5sum changed - script: | From ea800a4032a367e5287fb8e2364ecf89e23f4e8c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 15:56:24 +0530 Subject: [PATCH 178/513] test mac signing --- release-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f1d4972c2..961ddf889 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1113,6 +1113,8 @@ stages: displayName: 'ESRP CodeSigning macOS AzCopy' inputs: ConnectedServiceName: 'PMC ESRP AzCopy Signing' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' Pattern: 'azcopy_darwin*' signConfigType: inlineSignParams @@ -1122,7 +1124,9 @@ stages: { "KeyCode" : "CP-401337-Apple", "OperationCode" : "MacAppDeveloperSign", - "Parameters" : {}, + "Parameters":{ + "Hardening":"--options=runtime" + }, "ToolName" : "sign", "ToolVersion" : "1.0" } From 3cec8d265ea8d6b7d8ad855a6cefb46e38f3a262 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 16:27:44 +0530 Subject: [PATCH 179/513] test packages release --- common/version.go | 2 +- packages.csv | 25 +++++++++++++++++++++++- release-pipelines.yml | 44 +++++++++++++++++++++---------------------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/common/version.go b/common/version.go index 056c3f5d9..317391852 100644 --- a/common/version.go +++ b/common/version.go @@ -1,6 +1,6 @@ package common -const AzcopyVersion = "10.25.0" +const AzcopyVersion = "10.25.0~preview.1" const UserAgent = "AzCopy/" + AzcopyVersion const S3ImportUserAgent = "S3Import " + UserAgent const GCPImportUserAgent = "GCPImport " + UserAgent diff --git a/packages.csv b/packages.csv index f744f8fde..4b578b5eb 100644 --- a/packages.csv +++ b/packages.csv @@ -1,3 +1,26 @@ # Do not remove these comments # Format of the file : ,,, -Ubuntu-20.04,azcopyAmdDeb,microsoft-ubuntu-focal-prod-apt,testing +Ubuntu-18.04,fuse2AmdDeb,microsoft-ubuntu-bionic-prod-apt,bionic +Ubuntu-20.04,fuse3AmdDeb,microsoft-ubuntu-focal-prod-apt,focal +Ubuntu-22.04,fuse3AmdDeb,microsoft-ubuntu-jammy-prod-apt,jammy +Ubuntu-22.04,fuse3ArmDeb,microsoft-ubuntu-jammy-prod-apt,jammy +Ubuntu-24.04,fuse3AmdDeb,microsoft-ubuntu-noble-prod-apt,noble +Ubuntu-24.04,fuse3ArmDeb,microsoft-ubuntu-noble-prod-apt,noble +Debian-9.0,fuse2AmdDeb,microsoft-debian-stretch-prod-apt,stretch +Debian-10.0,fuse2AmdDeb,microsoft-debian-buster-prod-apt,buster +Debian-11.0,fuse3AmdDeb,microsoft-debian-bullseye-prod-apt,bullseye +RHEL-7.5,fuse3AmdRpm,microsoft-rhel7.5-prod-yum, +RHEL-7.8,fuse3AmdRpm,microsoft-rhel7.8-prod-yum, +RHEL-8.1,fuse3AmdRpm,microsoft-rhel8.1-prod-yum, +RHEL-8.2,fuse3AmdRpm,microsoft-rhel8.2-prod-yum, +RHEL-7.0,fuse3AmdRpm,microsoft-rhel7.0-prod-yum, +RHEL-8.0,fuse3AmdRpm,microsoft-rhel8.0-prod-yum, +RHEL-9.0,fuse3AmdRpm,microsoft-rhel9.0-prod-yum, +RHEL-9.0,fuse3ArmRpm,microsoft-rhel9.0-prod-yum, +CentOS-7.0,fuse3AmdRpm,microsoft-centos7-prod-yum, +CentOS-8.0,fuse3AmdRpm,microsoft-centos8-prod-yum, +SUSE-15Gen2,fuse3AmdRpm,microsoft-sles15-prod-yum, +Mariner-2.0-x86_64,marinerFuse3AmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, +Mariner-2.0-aarch64,marinerFuse3AarchRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, +Rocky-8.0,fuse3AmdRpm,microsoft-el8-prod-yum, +Rocky-9.0,fuse3AmdRpm,microsoft-el9-prod-yum, \ No newline at end of file diff --git a/release-pipelines.yml b/release-pipelines.yml index 961ddf889..fd7884094 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1109,28 +1109,28 @@ stages: displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning macOS AzCopy' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "CP-401337-Apple", - "OperationCode" : "MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning macOS AzCopy' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "CP-401337-Apple", + # "OperationCode" : "MacAppDeveloperSign", + # "Parameters":{ + # "Hardening":"--options=runtime" + # }, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 From 8315f60a017a994c78cd34c62ffb97fe7a8cf605 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 16:46:16 +0530 Subject: [PATCH 180/513] test packages release --- common/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/version.go b/common/version.go index 317391852..f9a8525bf 100644 --- a/common/version.go +++ b/common/version.go @@ -1,6 +1,6 @@ package common -const AzcopyVersion = "10.25.0~preview.1" +const AzcopyVersion = "10.25.0~preview.2" const UserAgent = "AzCopy/" + AzcopyVersion const S3ImportUserAgent = "S3Import " + UserAgent const GCPImportUserAgent = "GCPImport " + UserAgent From 9fc59e5c357b4a77c641cbc9c4061c516fee7b0d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 19:10:27 +0530 Subject: [PATCH 181/513] test packages release --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index fd7884094..6eccf7a99 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1707,8 +1707,8 @@ stages: cd mariner arm64file=$(ls azcopy*.arm64.rpm) amd64file=$(ls azcopy*.x86_64.rpm) - marinerArmFileName="${arm64file/-1/-1-cm2}" - marinerAmdFileName="${amd64file/-1/-1-cm2}" + marinerArmFileName="${arm64file/.1/.1-cm2}" + marinerAmdFileName="${amd64file/.1/.1-cm2}" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" ls -lRt From 95b1af3f4362f0a823a71843f4479c2358f29420 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 11 Jun 2024 19:10:27 +0530 Subject: [PATCH 182/513] test packages release --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index fd7884094..a4caa9a90 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1702,13 +1702,13 @@ stages: inputs: artifactName: 'azCopy-linux-signed' downloadPath: $(Build.ArtifactStagingDirectory) - + - script: | cd mariner arm64file=$(ls azcopy*.arm64.rpm) amd64file=$(ls azcopy*.x86_64.rpm) - marinerArmFileName="${arm64file/-1/-1-cm2}" - marinerAmdFileName="${amd64file/-1/-1-cm2}" + marinerArmFileName="${filename/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${filename/.x86_64.rpm/-cm2.x86_64.rpm}" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" ls -lRt From 7a9582a7372f5e836ee6a334c841ea5d47d4a880 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 10:22:55 +0530 Subject: [PATCH 183/513] test packages --- release-pipelines.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 76e84db3c..a4caa9a90 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1707,13 +1707,8 @@ stages: cd mariner arm64file=$(ls azcopy*.arm64.rpm) amd64file=$(ls azcopy*.x86_64.rpm) -<<<<<<< HEAD marinerArmFileName="${filename/.arm64.rpm/-cm2.arm64.rpm}" marinerAmdFileName="${filename/.x86_64.rpm/-cm2.x86_64.rpm}" -======= - marinerArmFileName="${arm64file/.1/.1-cm2}" - marinerAmdFileName="${amd64file/.1/.1-cm2}" ->>>>>>> 9fc59e5c357b4a77c641cbc9c4061c516fee7b0d mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" ls -lRt From 88bde87e580ebc3e7ee9c7b894de91489cb5e32b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 11:04:42 +0530 Subject: [PATCH 184/513] test packages --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a4caa9a90..4955d0604 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1707,8 +1707,8 @@ stages: cd mariner arm64file=$(ls azcopy*.arm64.rpm) amd64file=$(ls azcopy*.x86_64.rpm) - marinerArmFileName="${filename/.arm64.rpm/-cm2.arm64.rpm}" - marinerAmdFileName="${filename/.x86_64.rpm/-cm2.x86_64.rpm}" + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" ls -lRt @@ -1739,10 +1739,10 @@ stages: - script: | marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.arm64.rpm` - echo "Azcopy AMD DEB ID: $marinerArmRpm" + echo "Azcopy mariner AMD DEB ID: $marinerArmRpm" marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` - echo "Azcopy AMD DEB ID: $marinerAmdRpm" + echo "Azcopy mariner AMD DEB ID: $marinerAmdRpm" azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" From fc0e342bff0d18dd77bedd7917e64ebfe2dbdbfb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 11:10:41 +0530 Subject: [PATCH 185/513] test mac signing --- release-pipelines.yml | 55 +++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4955d0604..8c3a007bd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1132,32 +1132,37 @@ stages: # } # ] + - task: UseDotNet@2 + displayName: 'Install .NET Core 6.0 runtime' + inputs: + packageType: 'runtime' + version: '6.x' - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Mac executable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - # Pattern: 'azcopy_darwin*' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "KeyCode":"CP-401337-Apple", - # "OperationCode":"MacAppDeveloperSign", - # "Parameters":{ - # "Hardening":"--options=runtime" - # }, - # "ToolName":"sign", - # "ToolVersion":"1.0" - # } - # ] - # SessionTimeout: 30 + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"CP-401337-Apple", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 # Validate signed images have md5sum changed - script: | From df6e065affe8b3cffa196cfb9b158e83c2a1a461 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 13:10:18 +0530 Subject: [PATCH 186/513] test mac signing --- release-pipelines.yml | 714 ++++++++++++++++++++++-------------------- 1 file changed, 378 insertions(+), 336 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8c3a007bd..ba7e924c4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -305,260 +305,260 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirAmd64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - displayName: 'Accumulate pkg files' + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirAmd64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ + # displayName: 'Accumulate pkg files' - - script: | - cp NOTICE.txt pkgDirAmd64/usr/bin/ - displayName: 'Copy NOTICE.txt' + # - script: | + # cp NOTICE.txt pkgDirAmd64/usr/bin/ + # displayName: 'Copy NOTICE.txt' - - script: | - ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) - azcopy_version=${raw_version_output[2]} - echo "Version : $azcopy_version" - displayName: 'Verify the generated build' + # - script: | + # ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + # raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) + # azcopy_version=${raw_version_output[2]} + # echo "Version : $azcopy_version" + # displayName: 'Verify the generated build' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' - - script: | - cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add SE binary to signing stage' + # - script: | + # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add SE binary to signing stage' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - displayName: 'Accumulate arm64 pkg files' + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirArm64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + # displayName: 'Accumulate arm64 pkg files' - - script: | - cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt' + # - script: | + # cp NOTICE.txt pkgDirArm64/usr/bin/ + # displayName: 'Copy NOTICE.txt' - - script: | - GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Verify the generated build' + # - script: | + # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + # displayName: 'Verify the generated build' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - echo "Version : $version" - echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + # echo "Version : $version" + # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' - - script: | - cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add SE binary to signing stage' + # - script: | + # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add SE binary to signing stage' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # - job: Set_3 # timeoutInMinutes: 120 @@ -817,120 +817,120 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + # # Send images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy linux' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - displayName: 'Make Artifacts executable' + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + # displayName: 'Make Artifacts executable' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' # - job: Set_2 # timeoutInMinutes: 120 @@ -1136,7 +1136,7 @@ stages: displayName: 'Install .NET Core 6.0 runtime' inputs: packageType: 'runtime' - version: '6.x' + version: '6.0.0' - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Mac executable' @@ -1163,6 +1163,9 @@ stages: } ] SessionTimeout: 30 + script: | + # Install .NET SDK + curl -L https://dot.net/v1/dotnet-install.sh | bash -s -- --version 6.0.0 # Validate signed images have md5sum changed - script: | @@ -1611,15 +1614,29 @@ stages: # download artifacts that need to be published - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' + displayName: 'Download Linux Build Artifacts' inputs: artifactName: 'azCopy-linux-signed' downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip displayName: 'List Artifacts' - script: | @@ -1633,11 +1650,19 @@ stages: displayName: "Update dependencies" - ${{ if eq(parameters.post_release, true) }}: + + - script: | + mv azCopy-linux-signed/azcopy_linux_se_amd64 . + mv azCopy-linux-signed/azcopy_linux_se_arm64 . + mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . + displayName: 'Move binaries from signed folders' + workingDirectory: $(Build.ArtifactStagingDirectory) + # add release tags & push to github #TODO: update github connection and repo name while merging to main - task: GithubRelease@1 inputs: - githubConnection: 'github.com_dphulkar-msft' + githubConnection: 'dphulkar-msft-github-rel' repositoryName: 'dphulkar-msft/azure-storage-azcopy' action: 'create' @@ -1649,12 +1674,21 @@ stages: assets: | $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} isPreRelease: ${{ parameters.prerelease }} assetUploadMode: replace + - script: | + mv azcopy_linux_se_amd64 azCopy-linux-signed/ + mv azcopy_linux_se_arm64 azCopy-linux-signed/ + mv azcopy_windows_v7_arm.exe azCopy-windows-signed/ + displayName: 'Move binaries from signed folders' + workingDirectory: $(Build.ArtifactStagingDirectory) + - ${{ if eq(parameters.publish_artifacts, true) }}: - stage: PublishArtifacts dependsOn: ReleaseArtifacts @@ -1743,23 +1777,31 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.arm64.rpm` - echo "Azcopy mariner AMD DEB ID: $marinerArmRpm" + for file in "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed"/*; + do + if [ -f "$file" ]; then + fileID=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $file` + echo "File: $file and File ID: $fileID" + fi + done + + # marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.arm64.rpm` + # echo "Azcopy mariner AMD DEB ID: $marinerArmRpm" - marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` - echo "Azcopy mariner AMD DEB ID: $marinerAmdRpm" + # marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` + # echo "Azcopy mariner AMD DEB ID: $marinerAmdRpm" - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + # azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + # echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - azcopyAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.rpm` - echo "Azcopy AMD DEB ID: $azcopyAmdRpm" + # azcopyAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.rpm` + # echo "Azcopy AMD DEB ID: $azcopyAmdRpm" - azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - echo "Azcopy AMD DEB ID: $azcopyArmDeb" + # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + # echo "Azcopy AMD DEB ID: $azcopyArmDeb" - azcopyArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.rpm` - echo "Azcopy AMD DEB ID: $azcopyArmRpm" + # azcopyArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.rpm` + # echo "Azcopy AMD DEB ID: $azcopyArmRpm" while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" From 883c188a923b82cff6dd62c6950d8ace06a673ac Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 13:23:06 +0530 Subject: [PATCH 187/513] test github release --- release-pipelines.yml | 494 +++++++++++++++++++++--------------------- 1 file changed, 247 insertions(+), 247 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ba7e924c4..a2b40a00e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -305,260 +305,260 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirAmd64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - # displayName: 'Accumulate pkg files' + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # - script: | - # cp NOTICE.txt pkgDirAmd64/usr/bin/ - # displayName: 'Copy NOTICE.txt' + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirAmd64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ + displayName: 'Accumulate pkg files' - # - script: | - # ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - # raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) - # azcopy_version=${raw_version_output[2]} - # echo "Version : $azcopy_version" - # displayName: 'Verify the generated build' + - script: | + cp NOTICE.txt pkgDirAmd64/usr/bin/ + displayName: 'Copy NOTICE.txt' + + - script: | + ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) + azcopy_version=${raw_version_output[2]} + echo "Version : $azcopy_version" + displayName: 'Verify the generated build' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' - # - script: | - # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add SE binary to signing stage' + - script: | + cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add SE binary to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt $(work_dir)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirArm64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - # displayName: 'Accumulate arm64 pkg files' + # place the generated binary files & any additional files in appropriate locations + - script: | + mkdir -p pkgDirArm64/usr/bin/ + cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + displayName: 'Accumulate arm64 pkg files' - # - script: | - # cp NOTICE.txt pkgDirArm64/usr/bin/ - # displayName: 'Copy NOTICE.txt' + - script: | + cp NOTICE.txt pkgDirArm64/usr/bin/ + displayName: 'Copy NOTICE.txt' - # - script: | - # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - # displayName: 'Verify the generated build' + - script: | + GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + displayName: 'Verify the generated build' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - # echo "Version : $version" - # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + echo "Version : $version" + echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - script: | - # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add SE binary to signing stage' + - script: | + cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add SE binary to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # - job: Set_3 # timeoutInMinutes: 120 @@ -1138,34 +1138,34 @@ stages: packageType: 'runtime' version: '6.0.0' - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 - script: | - # Install .NET SDK - curl -L https://dot.net/v1/dotnet-install.sh | bash -s -- --version 6.0.0 + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Mac executable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "KeyCode":"CP-401337-Apple", + # "OperationCode":"MacAppDeveloperSign", + # "Parameters":{ + # "Hardening":"--options=runtime" + # }, + # "ToolName":"sign", + # "ToolVersion":"1.0" + # } + # ] + # SessionTimeout: 30 + # script: | + # # Install .NET SDK + # curl -L https://dot.net/v1/dotnet-install.sh | bash -s -- --version 6.0.0 # Validate signed images have md5sum changed - script: | From 311efe0d562f4d29cdaaf8a2771a912a9fd79a06 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 13:24:30 +0530 Subject: [PATCH 188/513] test windows build --- release-pipelines.yml | 286 +++++++++++++++++++++--------------------- 1 file changed, 143 insertions(+), 143 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a2b40a00e..bc5a3e889 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -560,164 +560,164 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - # - name: windows_amd64 - # value: '$(work_dir)\azcopy_windows_amd64' - # - name: windows_386 - # value: '$(work_dir)\azcopy_windows_386' - # - name: archives - # value: $(root_dir)\archives + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)\azcopy_windows_amd64' + - name: windows_386 + value: '$(work_dir)\azcopy_windows_386' + - name: archives + value: $(root_dir)\archives - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) - # - script: | - # mkdir $(windows_amd64) - # mkdir $(windows_386) - # mkdir $(archives) - # displayName: 'Create required directories' + - script: | + mkdir $(windows_amd64) + mkdir $(windows_386) + mkdir $(archives) + displayName: 'Create required directories' - # - script: | - # go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 + - script: | + go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(windows_386)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 + - script: | + go build -o "$(windows_386)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + - script: | + go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # dir $(work_dir) - # dir $(windows_386) - # dir $(windows_amd64) - # displayName: 'List the binaries' + - script: | + dir $(work_dir) + dir $(windows_386) + dir $(windows_amd64) + displayName: 'List the binaries' - # - script: | - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_386) - # displayName: 'Copy Notice file' + - script: | + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_386) + displayName: 'Copy Notice file' - # - task: PowerShell@2 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # # Run azcopy --version and capture the output - # $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version - - # # Example output: "azcopy version 10.25.0-Preview-1" - # # Extract the version part using string manipulation - # if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { - # $version = $matches[1] - # Write-Output "Extracted version: $version" + - task: PowerShell@2 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + # Run azcopy --version and capture the output + $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version + + # Example output: "azcopy version 10.25.0-Preview-1" + # Extract the version part using string manipulation + if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { + $version = $matches[1] + Write-Output "Extracted version: $version" - # # Set the pipeline variable - # Write-Host "##vso[task.setvariable variable=azcopy_version]$version" - # } else { - # Write-Error "Unable to extract version from azcopy output" - # } - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true + # Set the pipeline variable + Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + } else { + Write-Error "Unable to extract version from azcopy output" + } - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true - # - task: PowerShell@2 - # displayName: 'Copy zip to staging directory' - # inputs: - # targetType: 'inline' - # script: | - # # Ensure the destination directory exists - # New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force - - # # Copy the zip files and executable to the artifact staging directory - # $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" - # $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" - # $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" - # $destinationDir = "$(Build.ArtifactStagingDirectory)" - - # # Check if the source files exist before copying - # if (Test-Path $sourceAmd64Zip) { - # Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop - # Write-Output "Copied $sourceAmd64Zip to $destinationDir" - # } else { - # Write-Error "File not found: $sourceAmd64Zip" - # } - - # if (Test-Path $source386Zip) { - # Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop - # Write-Output "Copied $source386Zip to $destinationDir" - # } else { - # Write-Error "File not found: $source386Zip" - # } - - # if (Test-Path $sourceArmExe) { - # Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop - # Write-Output "Copied $sourceArmExe to $destinationDir" - # } else { - # Write-Error "File not found: $sourceArmExe" - # } + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PowerShell@2 + displayName: 'Copy zip to staging directory' + inputs: + targetType: 'inline' + script: | + # Ensure the destination directory exists + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force + + # Copy the zip files and executable to the artifact staging directory + $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" + $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" + $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" + $destinationDir = "$(Build.ArtifactStagingDirectory)" + + # Check if the source files exist before copying + if (Test-Path $sourceAmd64Zip) { + Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceAmd64Zip to $destinationDir" + } else { + Write-Error "File not found: $sourceAmd64Zip" + } + + if (Test-Path $source386Zip) { + Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $source386Zip to $destinationDir" + } else { + Write-Error "File not found: $source386Zip" + } + + if (Test-Path $sourceArmExe) { + Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceArmExe to $destinationDir" + } else { + Write-Error "File not found: $sourceArmExe" + } + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_4 timeoutInMinutes: 360 From 62ea3fab625689371356235f979401fd64b466ce Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 14:07:33 +0530 Subject: [PATCH 189/513] test github release --- release-pipelines.yml | 499 +++++++++++++++++++++--------------------- 1 file changed, 249 insertions(+), 250 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index bc5a3e889..8278c1f4c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -560,164 +560,164 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - - name: windows_amd64 - value: '$(work_dir)\azcopy_windows_amd64' - - name: windows_386 - value: '$(work_dir)\azcopy_windows_386' - - name: archives - value: $(root_dir)\archives + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + # - name: windows_amd64 + # value: '$(work_dir)\azcopy_windows_amd64' + # - name: windows_386 + # value: '$(work_dir)\azcopy_windows_386' + # - name: archives + # value: $(root_dir)\archives - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - script: | - mkdir $(windows_amd64) - mkdir $(windows_386) - mkdir $(archives) - displayName: 'Create required directories' + # - script: | + # mkdir $(windows_amd64) + # mkdir $(windows_386) + # mkdir $(archives) + # displayName: 'Create required directories' - - script: | - go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(windows_386)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(windows_386)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - dir $(work_dir) - dir $(windows_386) - dir $(windows_amd64) - displayName: 'List the binaries' + # - script: | + # dir $(work_dir) + # dir $(windows_386) + # dir $(windows_amd64) + # displayName: 'List the binaries' - - script: | - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_386) - displayName: 'Copy Notice file' + # - script: | + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_386) + # displayName: 'Copy Notice file' - - task: PowerShell@2 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - # Run azcopy --version and capture the output - $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version - - # Example output: "azcopy version 10.25.0-Preview-1" - # Extract the version part using string manipulation - if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { - $version = $matches[1] - Write-Output "Extracted version: $version" + # - task: PowerShell@2 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # # Run azcopy --version and capture the output + # $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version + + # # Example output: "azcopy version 10.25.0-Preview-1" + # # Extract the version part using string manipulation + # if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { + # $version = $matches[1] + # Write-Output "Extracted version: $version" - # Set the pipeline variable - Write-Host "##vso[task.setvariable variable=azcopy_version]$version" - } else { - Write-Error "Unable to extract version from azcopy output" - } - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true + # # Set the pipeline variable + # Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + # } else { + # Write-Error "Unable to extract version from azcopy output" + # } + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true - - task: PowerShell@2 - displayName: 'Copy zip to staging directory' - inputs: - targetType: 'inline' - script: | - # Ensure the destination directory exists - New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force - - # Copy the zip files and executable to the artifact staging directory - $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" - $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" - $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" - $destinationDir = "$(Build.ArtifactStagingDirectory)" - - # Check if the source files exist before copying - if (Test-Path $sourceAmd64Zip) { - Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceAmd64Zip to $destinationDir" - } else { - Write-Error "File not found: $sourceAmd64Zip" - } - - if (Test-Path $source386Zip) { - Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $source386Zip to $destinationDir" - } else { - Write-Error "File not found: $source386Zip" - } - - if (Test-Path $sourceArmExe) { - Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceArmExe to $destinationDir" - } else { - Write-Error "File not found: $sourceArmExe" - } + # - task: PowerShell@2 + # displayName: 'Copy zip to staging directory' + # inputs: + # targetType: 'inline' + # script: | + # # Ensure the destination directory exists + # New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force + + # # Copy the zip files and executable to the artifact staging directory + # $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" + # $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" + # $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" + # $destinationDir = "$(Build.ArtifactStagingDirectory)" + + # # Check if the source files exist before copying + # if (Test-Path $sourceAmd64Zip) { + # Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop + # Write-Output "Copied $sourceAmd64Zip to $destinationDir" + # } else { + # Write-Error "File not found: $sourceAmd64Zip" + # } + + # if (Test-Path $source386Zip) { + # Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop + # Write-Output "Copied $source386Zip to $destinationDir" + # } else { + # Write-Error "File not found: $source386Zip" + # } + + # if (Test-Path $sourceArmExe) { + # Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop + # Write-Output "Copied $sourceArmExe to $destinationDir" + # } else { + # Write-Error "File not found: $sourceArmExe" + # } - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_4 timeoutInMinutes: 360 @@ -817,120 +817,120 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # Send images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy linux' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' # - job: Set_2 # timeoutInMinutes: 120 @@ -1654,7 +1654,7 @@ stages: - script: | mv azCopy-linux-signed/azcopy_linux_se_amd64 . mv azCopy-linux-signed/azcopy_linux_se_arm64 . - mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . + # mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . displayName: 'Move binaries from signed folders' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1671,10 +1671,9 @@ stages: title: ${{ parameters.tag }} tag: ${{ parameters.tag }} - + #$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* assets: | $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' @@ -1685,7 +1684,7 @@ stages: - script: | mv azcopy_linux_se_amd64 azCopy-linux-signed/ mv azcopy_linux_se_arm64 azCopy-linux-signed/ - mv azcopy_windows_v7_arm.exe azCopy-windows-signed/ + # mv azcopy_windows_v7_arm.exe azCopy-windows-signed/ displayName: 'Move binaries from signed folders' workingDirectory: $(Build.ArtifactStagingDirectory) From df3acf593270d4d9eab544399bb1b5301dce980c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 14:22:22 +0530 Subject: [PATCH 190/513] test windows build --- release-pipelines.yml | 289 +++++++++++++++++++++--------------------- 1 file changed, 146 insertions(+), 143 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8278c1f4c..afc5f216e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -560,164 +560,167 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - # - name: windows_amd64 - # value: '$(work_dir)\azcopy_windows_amd64' - # - name: windows_386 - # value: '$(work_dir)\azcopy_windows_386' - # - name: archives - # value: $(root_dir)\archives + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)\azcopy_windows_amd64' + - name: windows_386 + value: '$(work_dir)\azcopy_windows_386' + - name: archives + value: $(root_dir)\archives - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) - # - script: | - # mkdir $(windows_amd64) - # mkdir $(windows_386) - # mkdir $(archives) - # displayName: 'Create required directories' + - script: | + mkdir $(windows_amd64) + mkdir $(windows_386) + mkdir $(archives) + displayName: 'Create required directories' - # - script: | - # go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 + - script: | + go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(windows_386)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 + - script: | + go build -o "$(windows_386)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + - script: | + go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # dir $(work_dir) - # dir $(windows_386) - # dir $(windows_amd64) - # displayName: 'List the binaries' + - script: | + dir $(work_dir) + dir $(windows_386) + dir $(windows_amd64) + displayName: 'List the binaries' - # - script: | - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_386) - # displayName: 'Copy Notice file' + - script: | + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_386) + displayName: 'Copy Notice file' - # - task: PowerShell@2 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # # Run azcopy --version and capture the output - # $azcopyOutput = & $(windows_amd64)\azcopy_windows_amd64.exe --version - - # # Example output: "azcopy version 10.25.0-Preview-1" - # # Extract the version part using string manipulation - # if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)-") { - # $version = $matches[1] - # Write-Output "Extracted version: $version" + - task: PowerShell@2 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + # Run azcopy --version and capture the output + $azcopyOutput = & "$(windows_amd64)\azcopy_windows_amd64.exe" --version + + # Output the raw azcopy version output for debugging + Write-Output "Raw azcopy output: $azcopyOutput" + + # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" + # Extract the version part using string manipulation + if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { + $version = $matches[1] + Write-Output "Extracted version: $version" - # # Set the pipeline variable - # Write-Host "##vso[task.setvariable variable=azcopy_version]$version" - # } else { - # Write-Error "Unable to extract version from azcopy output" - # } - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true + # Set the pipeline variable + Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + } else { + Write-Error "Unable to extract version from azcopy output" + } - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true - # - task: PowerShell@2 - # displayName: 'Copy zip to staging directory' - # inputs: - # targetType: 'inline' - # script: | - # # Ensure the destination directory exists - # New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force - - # # Copy the zip files and executable to the artifact staging directory - # $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" - # $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" - # $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" - # $destinationDir = "$(Build.ArtifactStagingDirectory)" - - # # Check if the source files exist before copying - # if (Test-Path $sourceAmd64Zip) { - # Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop - # Write-Output "Copied $sourceAmd64Zip to $destinationDir" - # } else { - # Write-Error "File not found: $sourceAmd64Zip" - # } - - # if (Test-Path $source386Zip) { - # Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop - # Write-Output "Copied $source386Zip to $destinationDir" - # } else { - # Write-Error "File not found: $source386Zip" - # } - - # if (Test-Path $sourceArmExe) { - # Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop - # Write-Output "Copied $sourceArmExe to $destinationDir" - # } else { - # Write-Error "File not found: $sourceArmExe" - # } + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PowerShell@2 + displayName: 'Copy zip to staging directory' + inputs: + targetType: 'inline' + script: | + # Ensure the destination directory exists + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force + + # Copy the zip files and executable to the artifact staging directory + $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" + $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" + $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" + $destinationDir = "$(Build.ArtifactStagingDirectory)" + + # Check if the source files exist before copying + if (Test-Path $sourceAmd64Zip) { + Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceAmd64Zip to $destinationDir" + } else { + Write-Error "File not found: $sourceAmd64Zip" + } + + if (Test-Path $source386Zip) { + Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $source386Zip to $destinationDir" + } else { + Write-Error "File not found: $source386Zip" + } + + if (Test-Path $sourceArmExe) { + Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceArmExe to $destinationDir" + } else { + Write-Error "File not found: $sourceArmExe" + } + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_4 timeoutInMinutes: 360 From 1f6fb0099e3e39d48e023991d24b9429e1dd008a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 14:46:11 +0530 Subject: [PATCH 191/513] test github release and publish --- release-pipelines.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index afc5f216e..27e909489 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1657,7 +1657,7 @@ stages: - script: | mv azCopy-linux-signed/azcopy_linux_se_amd64 . mv azCopy-linux-signed/azcopy_linux_se_arm64 . - # mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . + mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . displayName: 'Move binaries from signed folders' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1674,10 +1674,11 @@ stages: title: ${{ parameters.tag }} tag: ${{ parameters.tag }} - #$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + assets: | $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} @@ -1849,12 +1850,12 @@ stages: # - task: DownloadBuildArtifacts@0 # displayName: 'Download Build Artifacts' # inputs: - # artifactName: 'azCopy' + # artifactName: 'azCopy-linux-signed' # downloadPath: $(Build.ArtifactStagingDirectory) - # # install blobfuse2 + # # install azcopy # - script: | - # cd $(Build.ArtifactStagingDirectory)/azcopy + # cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed # ls | grep -i ubuntu-20.04 # sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y # azcopy version From 07806f79323f55db2bbf84b4367eb260487527a9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 14:46:11 +0530 Subject: [PATCH 192/513] test github release and publish --- release-pipelines.yml | 335 +++++++++++++++++++++--------------------- 1 file changed, 168 insertions(+), 167 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index afc5f216e..6f5c6e865 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -935,135 +935,135 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # # displayName: 'ESRP CodeSigning azCopy windows' - # # inputs: - # # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - # # Pattern: '*.zip, *.exe' - # # signConfigType: inlineSignParams - # # VerboseLogin: true - # # inlineOperation: | - # # [ - # # { - # # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # # "OperationCode" : "WindowsSign", - # # "Parameters" : {}, - # # "ToolName" : "sign", - # # "ToolVersion" : "1.0" - # # } - # # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Windows exectuable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - # Pattern: '*.exe, *.zip' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "keyCode": "CP-230012", - # "operationSetCode": "SigntoolSign", - # "parameters": [ - # { - # "parameterName": "OpusName", - # "parameterValue": "Microsoft" - # }, - # { - # "parameterName": "OpusInfo", - # "parameterValue": "http://www.microsoft.com" - # }, - # { - # "parameterName": "PageHash", - # "parameterValue": "/NPH" - # }, - # { - # "parameterName": "FileDigest", - # "parameterValue": "/fd sha256" - # }, - # { - # "parameterName": "TimeStamp", - # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - # } - # ], - # "toolName": "signtool.exe", - # "toolVersion": "6.2.9304.0" - # } - # ] - # SessionTimeout: 30 - - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy windows' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + # Pattern: '*.zip, *.exe' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "WindowsSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Windows exectuable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + # Pattern: '*.exe, *.zip' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "keyCode": "CP-230012", + # "operationSetCode": "SigntoolSign", + # "parameters": [ + # { + # "parameterName": "OpusName", + # "parameterValue": "Microsoft" + # }, + # { + # "parameterName": "OpusInfo", + # "parameterValue": "http://www.microsoft.com" + # }, + # { + # "parameterName": "PageHash", + # "parameterValue": "/NPH" + # }, + # { + # "parameterName": "FileDigest", + # "parameterValue": "/fd sha256" + # }, + # { + # "parameterName": "TimeStamp", + # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + # } + # ], + # "toolName": "signtool.exe", + # "toolVersion": "6.2.9304.0" + # } + # ] + # SessionTimeout: 30 + + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' - job: Set_3 timeoutInMinutes: 120 @@ -1657,7 +1657,7 @@ stages: - script: | mv azCopy-linux-signed/azcopy_linux_se_amd64 . mv azCopy-linux-signed/azcopy_linux_se_arm64 . - # mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . + mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . displayName: 'Move binaries from signed folders' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1674,10 +1674,11 @@ stages: title: ${{ parameters.tag }} tag: ${{ parameters.tag }} - #$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + assets: | $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* changeLogCompareToRelease: 'lastFullRelease' changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} @@ -1825,49 +1826,49 @@ stages: displayName: 'Publish the repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - # - stage: UpdateLatestVersion - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: UpdateVersion - # pool: - # vmImage: 'ubuntu-20.04' - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - stage: UpdateLatestVersion + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - ${{ if eq(parameters.update_version, true) }}: - # - script: | - # sudo apt-get install python3 -y - # python3 --version - # displayName: 'Installing Python' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy' - # downloadPath: $(Build.ArtifactStagingDirectory) + steps: + - checkout: none + + - ${{ if eq(parameters.update_version, true) }}: + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # # install blobfuse2 - # - script: | - # cd $(Build.ArtifactStagingDirectory)/azcopy - # ls | grep -i ubuntu-20.04 - # sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y - # azcopy version - # displayName: 'Installing azcopy' - - # - script: | - # wget https://raw.githubusercontent.com/Azure/azure-storage-storage/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py - # ls -l - # displayName: 'Getting Python script' - # workingDirectory: $(root_dir) - - # - script: | - # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy version`" - # displayName: 'Updating version number' - # workingDirectory: $(root_dir) + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.rpm + sudo apt-get install ./`ls azcopy*.arm64.rpm` -y + azcopy --version + displayName: 'Installing azcopy' + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-storage/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) From 27381a3a3d5558bbcae7e49ce719493c7a237793 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 16:21:41 +0530 Subject: [PATCH 193/513] test github release and publish --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6f5c6e865..f38c51b50 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1672,8 +1672,8 @@ stages: target: '$(Build.SourceVersion)' tagSource: 'userSpecifiedTag' - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} + title: 'azcopy-10.25.0' + tag: 'azcopy-10.25.0' assets: | $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* From cfd46cb2f27ca2058bebc4d0b389d33d138c264f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 12 Jun 2024 16:21:41 +0530 Subject: [PATCH 194/513] test github release and publish --- release-pipelines.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6f5c6e865..2de046305 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1111,6 +1111,11 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" + - task: UseDotNet@2 + displayName: 'Install .NET Core 6.0 runtime' + inputs: + packageType: 'runtime' + version: '6.0.0' # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 # displayName: 'ESRP CodeSigning macOS AzCopy' @@ -1135,11 +1140,7 @@ stages: # } # ] - - task: UseDotNet@2 - displayName: 'Install .NET Core 6.0 runtime' - inputs: - packageType: 'runtime' - version: '6.0.0' + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 # displayName: 'Sign Mac executable' @@ -1780,6 +1781,8 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | + mv azcopy_linux_se_amd64 ../ + mv azcopy_linux_se_arm64 ../ for file in "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed"/*; do if [ -f "$file" ]; then @@ -1810,6 +1813,9 @@ stages: echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) + + mv ../azcopy_linux_se_amd64 . + mv ../azcopy_linux_se_arm64 . displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ From e461ef0262256171efc8ad2d938f14b0ceb473ad Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 11:10:57 +0530 Subject: [PATCH 195/513] test windows signing --- release-pipelines.yml | 143 +++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 92 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 02fbd4646..e297a7431 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -977,73 +977,58 @@ stages: wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy windows' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - # Pattern: '*.zip, *.exe' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "WindowsSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Windows exectuable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - # Pattern: '*.exe, *.zip' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "keyCode": "CP-230012", - # "operationSetCode": "SigntoolSign", - # "parameters": [ - # { - # "parameterName": "OpusName", - # "parameterValue": "Microsoft" - # }, - # { - # "parameterName": "OpusInfo", - # "parameterValue": "http://www.microsoft.com" - # }, - # { - # "parameterName": "PageHash", - # "parameterValue": "/NPH" - # }, - # { - # "parameterName": "FileDigest", - # "parameterValue": "/fd sha256" - # }, - # { - # "parameterName": "TimeStamp", - # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - # } - # ], - # "toolName": "signtool.exe", - # "toolVersion": "6.2.9304.0" - # } - # ] - # SessionTimeout: 30 + + - task: UseDotNet@2 + displayName: 'Install .NET Core 6.0 runtime' + inputs: + packageType: 'runtime' + version: '6.0.0' + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows exectuable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + Pattern: '*.exe, *.zip' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 # Validate signed images have md5sum changed @@ -1108,7 +1093,6 @@ stages: sudo dpkg -i packages-microsoft-prod.deb sudo apt update sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - task: UseDotNet@2 @@ -1117,31 +1101,6 @@ stages: packageType: 'runtime' version: '6.0.0' - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning macOS AzCopy' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - # Pattern: 'azcopy_darwin*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "CP-401337-Apple", - # "OperationCode" : "MacAppDeveloperSign", - # "Parameters":{ - # "Hardening":"--options=runtime" - # }, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Mac executable' inputs: From 23ca17f393d4aba298dfc98f6b4eaa1328bd5366 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 11:42:01 +0530 Subject: [PATCH 196/513] test windows signing --- release-pipelines.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e297a7431..d41f9d7a4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -977,13 +977,14 @@ stages: wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - task: UseDotNet@2 - displayName: 'Install .NET Core 6.0 runtime' - inputs: - packageType: 'runtime' - version: '6.0.0' + # - task: UseDotNet@2 + # displayName: 'Install .NET Core 6.0 runtime' + # inputs: + # packageType: 'runtime' + # version: '6.0.0' - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Windows exectuable' @@ -1093,13 +1094,14 @@ stages: sudo dpkg -i packages-microsoft-prod.deb sudo apt update sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - task: UseDotNet@2 - displayName: 'Install .NET Core 6.0 runtime' - inputs: - packageType: 'runtime' - version: '6.0.0' + # - task: UseDotNet@2 + # displayName: 'Install .NET Core 6.0 runtime' + # inputs: + # packageType: 'runtime' + # version: '6.0.0' - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Mac executable' From 1015e2ea400fd4edd28c4667959f980f5c86c06e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 12:39:38 +0530 Subject: [PATCH 197/513] test windows signing --- release-pipelines.yml | 241 ++++++++++++++++++++++-------------------- 1 file changed, 128 insertions(+), 113 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d41f9d7a4..fbe11ce8e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -573,14 +573,6 @@ stages: variables: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' - - name: windows_amd64 - value: '$(work_dir)\azcopy_windows_amd64' - - name: windows_386 - value: '$(work_dir)\azcopy_windows_386' - - name: archives - value: $(root_dir)\archives steps: - task: GoTool@0 @@ -596,13 +588,7 @@ stages: workingDirectory: $(root_dir) - script: | - mkdir $(windows_amd64) - mkdir $(windows_386) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - go build -o "$(windows_amd64)\azcopy_windows_amd64.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" displayName: 'Generate Windows AMD64' env: GOARCH: amd64 @@ -610,7 +596,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(windows_386)\azcopy_windows_386.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" displayName: 'Generate Windows i386' env: GOARCH: 386 @@ -618,7 +604,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(work_dir)\azcopy_windows_v7_arm.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" displayName: 'Generate Windows ARM' env: GOARCH: arm @@ -626,90 +612,6 @@ stages: GOOS: windows CGO_ENABLED: 0 - - script: | - dir $(work_dir) - dir $(windows_386) - dir $(windows_amd64) - displayName: 'List the binaries' - - - script: | - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_386) - displayName: 'Copy Notice file' - - - task: PowerShell@2 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - # Run azcopy --version and capture the output - $azcopyOutput = & "$(windows_amd64)\azcopy_windows_amd64.exe" --version - - # Output the raw azcopy version output for debugging - Write-Output "Raw azcopy output: $azcopyOutput" - - # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" - # Extract the version part using string manipulation - if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { - $version = $matches[1] - Write-Output "Extracted version: $version" - - # Set the pipeline variable - Write-Host "##vso[task.setvariable variable=azcopy_version]$version" - } else { - Write-Error "Unable to extract version from azcopy output" - } - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - task: PowerShell@2 - displayName: 'Copy zip to staging directory' - inputs: - targetType: 'inline' - script: | - # Ensure the destination directory exists - New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force - - # Copy the zip files and executable to the artifact staging directory - $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" - $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" - $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" - $destinationDir = "$(Build.ArtifactStagingDirectory)" - - # Check if the source files exist before copying - if (Test-Path $sourceAmd64Zip) { - Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceAmd64Zip to $destinationDir" - } else { - Write-Error "File not found: $sourceAmd64Zip" - } - - if (Test-Path $source386Zip) { - Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $source386Zip to $destinationDir" - } else { - Write-Error "File not found: $source386Zip" - } - - if (Test-Path $sourceArmExe) { - Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceArmExe to $destinationDir" - } else { - Write-Error "File not found: $sourceArmExe" - } - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-windows-temp' @@ -949,6 +851,14 @@ stages: variables: - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)\azcopy_windows_amd64' + - name: windows_386 + value: '$(work_dir)\azcopy_windows_386' + - name: archives + value: $(root_dir)\archives steps: - checkout: none @@ -980,12 +890,6 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - # - task: UseDotNet@2 - # displayName: 'Install .NET Core 6.0 runtime' - # inputs: - # packageType: 'runtime' - # version: '6.0.0' - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Windows exectuable' inputs: @@ -996,12 +900,12 @@ stages: AuthCertName: 'azcopy-auth' AuthSignCertName: 'azcopy-sign-test' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' - Pattern: '*.exe, *.zip' + Pattern: '*.exe' signConfigType: inlineSignParams inlineOperation: | [ { - "keyCode": "CP-230012", + "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", "operationSetCode": "SigntoolSign", "parameters": [ { @@ -1031,15 +935,100 @@ stages: ] SessionTimeout: 30 - # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ + #mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ + #rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ displayName: 'Make Artifacts executable' + - script: | + mkdir $(windows_amd64) + mkdir $(windows_386) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_386) + displayName: 'Copy required files' + + - task: PowerShell@2 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + # Run azcopy --version and capture the output + $azcopyOutput = & "$(windows_amd64)\azcopy_windows_amd64.exe" --version + + # Output the raw azcopy version output for debugging + Write-Output "Raw azcopy output: $azcopyOutput" + + # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" + # Extract the version part using string manipulation + if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { + $version = $matches[1] + Write-Output "Extracted version: $version" + + # Set the pipeline variable + Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + } else { + Write-Error "Unable to extract version from azcopy output" + } + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - task: PowerShell@2 + displayName: 'Copy zip to staging directory' + inputs: + targetType: 'inline' + script: | + # Ensure the destination directory exists + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force + + # Copy the zip files and executable to the artifact staging directory + $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" + $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" + $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" + $destinationDir = "$(Build.ArtifactStagingDirectory)" + + # Check if the source files exist before copying + if (Test-Path $sourceAmd64Zip) { + Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceAmd64Zip to $destinationDir" + } else { + Write-Error "File not found: $sourceAmd64Zip" + } + + if (Test-Path $source386Zip) { + Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $source386Zip to $destinationDir" + } else { + Write-Error "File not found: $source386Zip" + } + + if (Test-Path $sourceArmExe) { + Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop + Write-Output "Copied $sourceArmExe to $destinationDir" + } else { + Write-Error "File not found: $sourceArmExe" + } + - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* @@ -1118,7 +1107,7 @@ stages: inlineOperation: | [ { - "KeyCode":"CP-401337-Apple", + "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", "OperationCode":"MacAppDeveloperSign", "Parameters":{ "Hardening":"--options=runtime" @@ -1129,6 +1118,32 @@ stages: ] SessionTimeout: 30 + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Notarize Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"CP-401337-Apple", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* From b70ac1b8e3513913e7136691ad3f67d3ccedf2ae Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 12:40:48 +0530 Subject: [PATCH 198/513] test windows signing --- release-pipelines.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index fbe11ce8e..e05ca89c1 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1131,17 +1131,17 @@ stages: Pattern: 'azcopy_darwin*' signConfigType: inlineSignParams inlineOperation: | - [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppNotarize", - "Parameters":{ - "BundleId":"com.microsoft.AzCopy" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] + [ + { + "KeyCode":"CP-401337-Apple", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] SessionTimeout: 30 # Validate signed images have md5sum changed From 96d7402e66322a6656a25ec0b699675aad1d23ec Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 13:02:17 +0530 Subject: [PATCH 199/513] test windows signing --- release-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e05ca89c1..52018799d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -861,8 +861,6 @@ stages: value: $(root_dir)\archives steps: - - checkout: none - - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' From d8dfe83376929bcb79adf53f59af560496ac74e7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 13:41:03 +0530 Subject: [PATCH 200/513] test windows signing --- release-pipelines.yml | 87 +++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 58 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 52018799d..3585cff06 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -612,6 +612,30 @@ stages: GOOS: windows CGO_ENABLED: 0 + - task: PowerShell@2 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + # Run azcopy --version and capture the output + $azcopyOutput = & "$(windows_amd64)\azcopy_windows_amd64.exe" --version + + # Output the raw azcopy version output for debugging + Write-Output "Raw azcopy output: $azcopyOutput" + + # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" + # Extract the version part using string manipulation + if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { + $version = $matches[1] + Write-Output "Extracted version: $version" + + # Set the pipeline variable + Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + } else { + Write-Error "Unable to extract version from azcopy output" + } + + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-windows-temp' @@ -953,30 +977,7 @@ stages: cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_386) displayName: 'Copy required files' - - - task: PowerShell@2 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - # Run azcopy --version and capture the output - $azcopyOutput = & "$(windows_amd64)\azcopy_windows_amd64.exe" --version - - # Output the raw azcopy version output for debugging - Write-Output "Raw azcopy output: $azcopyOutput" - - # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" - # Extract the version part using string manipulation - if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { - $version = $matches[1] - Write-Output "Extracted version: $version" - - # Set the pipeline variable - Write-Host "##vso[task.setvariable variable=azcopy_version]$version" - } else { - Write-Error "Unable to extract version from azcopy output" - } - + - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit Build' inputs: @@ -991,41 +992,11 @@ stages: archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' continueOnError: true - - task: PowerShell@2 + - script: | + cp $(archives)\azcopy_windows_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)\azcopy_windows_386_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(work_dir)\azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - inputs: - targetType: 'inline' - script: | - # Ensure the destination directory exists - New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)" -Force - - # Copy the zip files and executable to the artifact staging directory - $sourceAmd64Zip = "$(archives)\azcopy_windows_amd64_$(azcopy_version).zip" - $source386Zip = "$(archives)\azcopy_windows_386_$(azcopy_version).zip" - $sourceArmExe = "$(work_dir)\azcopy_windows_v7_arm.exe" - $destinationDir = "$(Build.ArtifactStagingDirectory)" - - # Check if the source files exist before copying - if (Test-Path $sourceAmd64Zip) { - Copy-Item -Path $sourceAmd64Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceAmd64Zip to $destinationDir" - } else { - Write-Error "File not found: $sourceAmd64Zip" - } - - if (Test-Path $source386Zip) { - Copy-Item -Path $source386Zip -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $source386Zip to $destinationDir" - } else { - Write-Error "File not found: $source386Zip" - } - - if (Test-Path $sourceArmExe) { - Copy-Item -Path $sourceArmExe -Destination $destinationDir -Force -ErrorAction Stop - Write-Output "Copied $sourceArmExe to $destinationDir" - } else { - Write-Error "File not found: $sourceArmExe" - } - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) From 3c7d37e4a420271f27d7f86ead715ab3705184c7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 14:09:14 +0530 Subject: [PATCH 201/513] test windows signing --- release-pipelines.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 3585cff06..32154a1af 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -618,7 +618,7 @@ stages: targetType: 'inline' script: | # Run azcopy --version and capture the output - $azcopyOutput = & "$(windows_amd64)\azcopy_windows_amd64.exe" --version + $azcopyOutput = & "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" --version # Output the raw azcopy version output for debugging Write-Output "Raw azcopy output: $azcopyOutput" @@ -630,7 +630,7 @@ stages: Write-Output "Extracted version: $version" # Set the pipeline variable - Write-Host "##vso[task.setvariable variable=azcopy_version]$version" + #Write-Host "##vso[task.setvariable variable=AZCOPY_VERSION;isOutput=true]$version" } else { Write-Error "Unable to extract version from azcopy output" } @@ -883,7 +883,9 @@ stages: value: '$(work_dir)\azcopy_windows_386' - name: archives value: $(root_dir)\archives - + - name: AZCOPY_VERSION + value: $[ dependencies.SetVariables.outputs['SetVars.AZCOPY_VERSION'] ] + steps: - script: | echo ${{ parameters.tag }} From cb4013d79f54512fd474ca8df1c7787e8d126e1c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 14:45:47 +0530 Subject: [PATCH 202/513] test windows signing --- release-pipelines.yml | 712 +++++++++++++++++++++--------------------- 1 file changed, 356 insertions(+), 356 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 32154a1af..0d5a6cdf7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -305,260 +305,260 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirAmd64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - displayName: 'Accumulate pkg files' - - - script: | - cp NOTICE.txt pkgDirAmd64/usr/bin/ - displayName: 'Copy NOTICE.txt' - - - script: | - ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) - azcopy_version=${raw_version_output[2]} - echo "Version : $azcopy_version" - displayName: 'Verify the generated build' + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirAmd64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ + # displayName: 'Accumulate pkg files' + + # - script: | + # cp NOTICE.txt pkgDirAmd64/usr/bin/ + # displayName: 'Copy NOTICE.txt' + + # - script: | + # ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version + # raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) + # azcopy_version=${raw_version_output[2]} + # echo "Version : $azcopy_version" + # displayName: 'Verify the generated build' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - script: | - cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add SE binary to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - script: | + # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add SE binary to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(work_dir)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDirArm64/usr/bin/ - cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - displayName: 'Accumulate arm64 pkg files' - - - script: | - cp NOTICE.txt pkgDirArm64/usr/bin/ - displayName: 'Copy NOTICE.txt' - - - script: | - GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - displayName: 'Verify the generated build' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - echo "Version : $version" - echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(work_dir)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # # place the generated binary files & any additional files in appropriate locations + # - script: | + # mkdir -p pkgDirArm64/usr/bin/ + # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ + # displayName: 'Accumulate arm64 pkg files' + + # - script: | + # cp NOTICE.txt pkgDirArm64/usr/bin/ + # displayName: 'Copy NOTICE.txt' + + # - script: | + # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version + # displayName: 'Verify the generated build' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + # echo "Version : $version" + # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - script: | - cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add SE binary to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - script: | + # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add SE binary to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -618,7 +618,7 @@ stages: targetType: 'inline' script: | # Run azcopy --version and capture the output - $azcopyOutput = & "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" --version + $azcopyOutput = & "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" --version # Output the raw azcopy version output for debugging Write-Output "Raw azcopy output: $azcopyOutput" @@ -746,120 +746,120 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # # Send images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy linux' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 From fb8685943cd1a4a40babc0a0590d276168bc792c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 15:25:20 +0530 Subject: [PATCH 203/513] test windows signing --- release-pipelines.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0d5a6cdf7..c89423508 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -984,20 +984,20 @@ stages: displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + archiveFile: '$(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + archiveFile: '$(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip' continueOnError: true - script: | - cp $(archives)\azcopy_windows_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)\azcopy_windows_386_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(work_dir)\azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - script: | @@ -1056,12 +1056,6 @@ stages: sudo apt install apt-transport-https -y sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - # - task: UseDotNet@2 - # displayName: 'Install .NET Core 6.0 runtime' - # inputs: - # packageType: 'runtime' - # version: '6.0.0' - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Mac executable' From e0e5b02a503cbd4f80acf5a3d398e1390abc014f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 18:14:33 +0530 Subject: [PATCH 204/513] test linux tar.gz creation --- release-pipelines.yml | 559 +++++++++++++++++++++++++----------------- 1 file changed, 339 insertions(+), 220 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c89423508..df7534eea 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -305,260 +305,379 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirAmd64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_amd64 pkgDirAmd64/usr/bin/ - # displayName: 'Accumulate pkg files' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - # - script: | - # cp NOTICE.txt pkgDirAmd64/usr/bin/ - # displayName: 'Copy NOTICE.txt' + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # - script: | - # ./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version - # raw_version_output=(`/pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version`) - # azcopy_version=${raw_version_output[2]} - # echo "Version : $azcopy_version" - # displayName: 'Verify the generated build' - + - script: | + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p /pkgDirAmd64/usr/bin/ + cp NOTICE.txt /pkgDirAmd64/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 /pkgDirAmd64/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy_linux_amd64 --version + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - # -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ + -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' - # - script: | - # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add SE binary to signing stage' + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r (archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(work_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(work_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' - # - script: | - # sudo ls -lRt $(work_dir)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # # place the generated binary files & any additional files in appropriate locations - # - script: | - # mkdir -p pkgDirArm64/usr/bin/ - # cp -r $(work_dir)/azcopy_linux_arm64 pkgDirArm64/usr/bin/ - # displayName: 'Accumulate arm64 pkg files' + - script: | + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' - # - script: | - # cp NOTICE.txt pkgDirArm64/usr/bin/ - # displayName: 'Copy NOTICE.txt' + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - # - script: | - # GOARCH=arm64 GOOS=linux ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version - # displayName: 'Verify the generated build' + - script: | + mkdir -p /pkgDirArm64/usr/bin/ + cp NOTICE.txt /pkgDirArm64/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 /pkgDirArm64/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + displayName: 'Copy required files' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - # echo "Version : $version" - # echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy_linux_arm64 --version + azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 + echo "Version : $version" + echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - # -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ + -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - script: | - # cp -r $(work_dir)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add SE binary to signing stage' + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 From e9f41764f8cc05086579e9578113f6e3c783bb4a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 18:27:43 +0530 Subject: [PATCH 205/513] test linux tar.gz creation --- release-pipelines.yml | 265 +++++++++++++++++++++--------------------- 1 file changed, 135 insertions(+), 130 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index df7534eea..abf55090a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -403,7 +403,7 @@ stages: displayName: "Remove binaries from staging directory" - script: | - mkdir -p /pkgDirAmd64/usr/bin/ + sudo mkdir -p /pkgDirAmd64/usr/bin/ cp NOTICE.txt /pkgDirAmd64/usr/bin/ cp $(linux_amd64)/azcopy_linux_amd64 /pkgDirAmd64/usr/bin/ cp NOTICE.txt $(linux_amd64)/ @@ -593,7 +593,7 @@ stages: displayName: "Remove binaries from staging directory" - script: | - mkdir -p /pkgDirArm64/usr/bin/ + sudo mkdir -p /pkgDirArm64/usr/bin/ cp NOTICE.txt /pkgDirArm64/usr/bin/ cp $(linux_arm64)/azcopy_linux_arm64 /pkgDirArm64/usr/bin/ cp NOTICE.txt $(linux_se_arm64)/ @@ -731,29 +731,34 @@ stages: GOOS: windows CGO_ENABLED: 0 - - task: PowerShell@2 - displayName: 'Extract AZCopy version' + - task: PublishBuildArtifacts@1 inputs: - targetType: 'inline' - script: | - # Run azcopy --version and capture the output - $azcopyOutput = & "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" --version - - # Output the raw azcopy version output for debugging - Write-Output "Raw azcopy output: $azcopyOutput" + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" - # Extract the version part using string manipulation - if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { - $version = $matches[1] - Write-Output "Extracted version: $version" + # - task: PowerShell@2 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # # Run azcopy --version and capture the output + # $azcopyOutput = & "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" --version + + # # Output the raw azcopy version output for debugging + # Write-Output "Raw azcopy output: $azcopyOutput" + + # # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" + # # Extract the version part using string manipulation + # if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { + # $version = $matches[1] + # Write-Output "Extracted version: $version" - # Set the pipeline variable - #Write-Host "##vso[task.setvariable variable=AZCOPY_VERSION;isOutput=true]$version" - } else { - Write-Error "Unable to extract version from azcopy output" - } - + # # Set the pipeline variable + # #Write-Host "##vso[task.setvariable variable=AZCOPY_VERSION;isOutput=true]$version" + # } else { + # Write-Error "Unable to extract version from azcopy output" + # } - task: PublishBuildArtifacts@1 inputs: @@ -865,120 +870,120 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # # Send images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy linux' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + Pattern: '*.rpm, *.deb, *tar.gz' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 From 8db27a6f54038d4ebe23fdd16b46f45911fa07e0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 18:37:10 +0530 Subject: [PATCH 206/513] test linux tar.gz creation --- release-pipelines.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index abf55090a..3a8a33328 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -404,8 +404,8 @@ stages: - script: | sudo mkdir -p /pkgDirAmd64/usr/bin/ - cp NOTICE.txt /pkgDirAmd64/usr/bin/ - cp $(linux_amd64)/azcopy_linux_amd64 /pkgDirAmd64/usr/bin/ + sudo cp NOTICE.txt /pkgDirAmd64/usr/bin/ + sudo cp $(linux_amd64)/azcopy_linux_amd64 /pkgDirAmd64/usr/bin/ cp NOTICE.txt $(linux_amd64)/ cp NOTICE.txt $(linux_se_amd64)/ displayName: 'Copy required files' @@ -594,8 +594,8 @@ stages: - script: | sudo mkdir -p /pkgDirArm64/usr/bin/ - cp NOTICE.txt /pkgDirArm64/usr/bin/ - cp $(linux_arm64)/azcopy_linux_arm64 /pkgDirArm64/usr/bin/ + sudo cp NOTICE.txt /pkgDirArm64/usr/bin/ + sudo cp $(linux_arm64)/azcopy_linux_arm64 /pkgDirArm64/usr/bin/ cp NOTICE.txt $(linux_se_arm64)/ displayName: 'Copy required files' @@ -820,6 +820,21 @@ stages: - script: | GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" - task: Bash@3 displayName: 'Extract AZCopy version' From 6907f4a5d484aecf203bf353ad619b3a966c5359 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 20:49:14 +0530 Subject: [PATCH 207/513] test linux tar.gz creation --- release-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 3a8a33328..f869bb1bd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -410,6 +410,10 @@ stages: cp NOTICE.txt $(linux_se_amd64)/ displayName: 'Copy required files' + - script: | + sudo ls -lRt /pkgDirAmd64/usr/bin/ + displayName: 'List the files' + - script: | export GOARCH=amd64 export GOOS=linux @@ -599,6 +603,10 @@ stages: cp NOTICE.txt $(linux_se_arm64)/ displayName: 'Copy required files' + - script: | + sudo ls -lRt /pkgDirAmd64/usr/bin/ + displayName: 'List the files' + - script: | export GOARCH=arm64 export GOOS=linux From 30e72212417560b97ddd8d41d1c0cc2f32c67cee Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 21:00:30 +0530 Subject: [PATCH 208/513] test linux tar.gz creation --- release-pipelines.yml | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f869bb1bd..6826bf3ae 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -403,15 +403,15 @@ stages: displayName: "Remove binaries from staging directory" - script: | - sudo mkdir -p /pkgDirAmd64/usr/bin/ - sudo cp NOTICE.txt /pkgDirAmd64/usr/bin/ - sudo cp $(linux_amd64)/azcopy_linux_amd64 /pkgDirAmd64/usr/bin/ + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ cp NOTICE.txt $(linux_amd64)/ cp NOTICE.txt $(linux_se_amd64)/ displayName: 'Copy required files' - script: | - sudo ls -lRt /pkgDirAmd64/usr/bin/ + sudo ls -lRt pkgDir/usr/bin/ displayName: 'List the files' - script: | @@ -444,23 +444,23 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirAmd64/ \ - -v `./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDirAmd64/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -597,14 +597,14 @@ stages: displayName: "Remove binaries from staging directory" - script: | - sudo mkdir -p /pkgDirArm64/usr/bin/ - sudo cp NOTICE.txt /pkgDirArm64/usr/bin/ - sudo cp $(linux_arm64)/azcopy_linux_arm64 /pkgDirArm64/usr/bin/ + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ cp NOTICE.txt $(linux_se_arm64)/ displayName: 'Copy required files' - script: | - sudo ls -lRt /pkgDirAmd64/usr/bin/ + sudo ls -lRt pkgDir/usr/bin/ displayName: 'List the files' - script: | @@ -635,26 +635,23 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ # TODO: Add description - script: | - fpm -s dir -t deb -n azcopy -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - version = ./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3 - echo "Version : $version" - echo `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDirArm64/ \ - -v `./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDirArm64/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' From 27647ccb7b255bfba9c23f636e6d9866203f5ec8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 21:13:01 +0530 Subject: [PATCH 209/513] test linux tar.gz creation --- release-pipelines.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6826bf3ae..037339c3c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -482,7 +482,7 @@ stages: continueOnError: true - script: | - cp -r (archives)/* $(Build.ArtifactStagingDirectory) + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Add tar.gz to signing stage' @@ -671,6 +671,11 @@ stages: archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' - task: PublishBuildArtifacts@1 inputs: @@ -1679,6 +1684,23 @@ stages: dependsOn: TestArtifacts condition: succeeded('TestArtifacts') jobs: + # - job: ReleaseToContainer + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + # variables: + # - name: Test_Release_Account_Name + # value: 'azcopyprivatedrops' + # - name: Release_Account_Name + # value: 'azcopyvnextrelease' + - job: ReleaseAzcopy timeoutInMinutes: 120 strategy: From 63246e70e6418fe14ed3f37ab4cb96281c076283 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 22:07:08 +0530 Subject: [PATCH 210/513] test linux tar.gz creation --- release-pipelines.yml | 126 ++++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 34 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 037339c3c..6ef8042e3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -421,24 +421,24 @@ stages: azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" else - echo "azcopy version string not found in output" + echo "Unable to extract version from azcopy output" exit 1 fi + else + echo "azcopy version string not found in output" + exit 1 + fi displayName: 'Verify and extract the generated build version' # using fpm tool for packaging of our binary & performing post-install operations @@ -1046,6 +1046,12 @@ stages: inputs: artifactName: 'azCopy-windows-temp' downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory)/azcopy_linux_amd64 - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) @@ -1133,19 +1139,19 @@ stages: displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip' + archiveFile: '$(archives)/azcopy_windows_amd64.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip' + archiveFile: '$(archives)/azcopy_windows_386.zip' continueOnError: true - script: | - cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_amd64.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386.zip $(Build.ArtifactStagingDirectory) cp $(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' @@ -1684,22 +1690,74 @@ stages: dependsOn: TestArtifacts condition: succeeded('TestArtifacts') jobs: - # - job: ReleaseToContainer - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - # variables: - # - name: Test_Release_Account_Name - # value: 'azcopyprivatedrops' - # - name: Release_Account_Name - # value: 'azcopyvnextrelease' + - job: ReleaseToContainer + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: AzureCLI@2 + displayName: 'Azure CLI ' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) + azcopy_version=${raw_version_output[2]} + + today=`date +"%Y%m%d"` + if [ $(Test_Mode) = "on" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy_linux_amd64 + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + - job: ReleaseAzcopy timeoutInMinutes: 120 From 310fe7c49d71a6743036a34e9d2f12c1df5a7515 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 13 Jun 2024 22:08:19 +0530 Subject: [PATCH 211/513] test linux tar.gz creation --- release-pipelines.yml | 126 +++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6ef8042e3..ae7e440d8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1690,73 +1690,73 @@ stages: dependsOn: TestArtifacts condition: succeeded('TestArtifacts') jobs: - - job: ReleaseToContainer - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - variables: - - name: Test_Release_Account_Name - value: 'azcopyprivatedrops' - - name: Release_Account_Name - value: 'azcopyvnextrelease' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - job: ReleaseToContainer + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + # variables: + # - name: Test_Release_Account_Name + # value: 'azcopyprivatedrops' + # - name: Release_Account_Name + # value: 'azcopyvnextrelease' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Linux Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Windows Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Mac Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: AzureCLI@2 - displayName: 'Azure CLI ' - inputs: - azureSubscription: 'ESRP KeyVault identity' - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - # Find the current version - raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) - azcopy_version=${raw_version_output[2]} + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Mac Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: AzureCLI@2 + # displayName: 'Azure CLI ' + # inputs: + # azureSubscription: 'ESRP KeyVault identity' + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # # Find the current version + # raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) + # azcopy_version=${raw_version_output[2]} - today=`date +"%Y%m%d"` - if [ $(Test_Mode) = "on" ]; then - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - else - container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - fi - echo "Artifacts will be uploaded to: $container_url" + # today=`date +"%Y%m%d"` + # if [ $(Test_Mode) = "on" ]; then + # container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + # else + # container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + # fi + # echo "Artifacts will be uploaded to: $container_url" - executable_name=azcopy_linux_amd64 + # executable_name=azcopy_linux_amd64 - # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + # # Upload the archived builds + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy From 8b197f6df10a15b35119467a6e92047394e0b841 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 10:31:37 +0530 Subject: [PATCH 212/513] test linux tar.gz creation --- release-pipelines.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ae7e440d8..39cced6ee 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1025,13 +1025,13 @@ stages: variables: - group: AZCOPY_ESRP_SECRET - name: work_dir - value: '$(System.DefaultWorkingDirectory)\azure-storage-azcopy' + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - name: windows_amd64 - value: '$(work_dir)\azcopy_windows_amd64' + value: '$(work_dir)/azcopy_windows_amd64' - name: windows_386 - value: '$(work_dir)\azcopy_windows_386' + value: '$(work_dir)/azcopy_windows_386' - name: archives - value: $(root_dir)\archives + value: $(root_dir)/archives - name: AZCOPY_VERSION value: $[ dependencies.SetVariables.outputs['SetVars.AZCOPY_VERSION'] ] @@ -1135,6 +1135,11 @@ stages: cp NOTICE.txt $(windows_386) displayName: 'Copy required files' + - script: | + sudo apt-get update + sudo apt-get install -y zip + displayName: 'Install zip utility' + - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit Build' inputs: From 15ed0093052bfddc0613b8058a0c0503829b5553 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 10:57:38 +0530 Subject: [PATCH 213/513] test linux tar.gz creation --- release-pipelines.yml | 172 ++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 83 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 39cced6ee..7a4e8ac42 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -8,6 +8,11 @@ parameters: displayName: 'New Release Tag' type: string default: 'azcopy-' + + - name: test_mode + displayName: 'Test Mode' + type: boolean + default: false - name: unit_test displayName: 'Execute Unit Tests' @@ -1031,7 +1036,7 @@ stages: - name: windows_386 value: '$(work_dir)/azcopy_windows_386' - name: archives - value: $(root_dir)/archives + value: '$(work_dir)/archives' - name: AZCOPY_VERSION value: $[ dependencies.SetVariables.outputs['SetVars.AZCOPY_VERSION'] ] @@ -1057,6 +1062,12 @@ stages: sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* displayName: 'List Artifacts' + + - script: | + mkdir $(windows_amd64) + mkdir $(windows_386) + mkdir $(archives) + displayName: 'Create required directories' - script: | sudo apt-get clean @@ -1118,16 +1129,8 @@ stages: - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - #mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* $(Build.ArtifactStagingDirectory)/ - #rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/ displayName: 'Make Artifacts executable' - - script: | - mkdir $(windows_amd64) - mkdir $(windows_386) - mkdir $(archives) - displayName: 'Create required directories' - - script: | cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) @@ -1135,23 +1138,18 @@ stages: cp NOTICE.txt $(windows_386) displayName: 'Copy required files' - - script: | - sudo apt-get update - sudo apt-get install -y zip - displayName: 'Install zip utility' - - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64.zip' + archiveFile: '$(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386.zip' + archiveFile: '$(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip' continueOnError: true - script: | @@ -1695,74 +1693,82 @@ stages: dependsOn: TestArtifacts condition: succeeded('TestArtifacts') jobs: - # - job: ReleaseToContainer - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - # variables: - # - name: Test_Release_Account_Name - # value: 'azcopyprivatedrops' - # - name: Release_Account_Name - # value: 'azcopyvnextrelease' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Linux Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Windows Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Mac Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Mac Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: AzureCLI@2 - # displayName: 'Azure CLI ' - # inputs: - # azureSubscription: 'ESRP KeyVault identity' - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # # Find the current version - # raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) - # azcopy_version=${raw_version_output[2]} - - # today=`date +"%Y%m%d"` - # if [ $(Test_Mode) = "on" ]; then - # container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - # else - # container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - # fi - # echo "Artifacts will be uploaded to: $container_url" - - # executable_name=azcopy_linux_amd64 - - # # Upload the archived builds - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + - job: ReleaseToContainer + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + + steps: + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: ArchiveFiles@2 + displayName: 'Archive all binaries to drop.zip' + inputs: + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + archiveFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip' + continueOnError: true + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) + azcopy_version=${raw_version_output[2]} + + today=`date +"%Y%m%d"` + if [ $(test_mode) = "on" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy_linux_amd64 + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/*.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From a8e674e124915e42b031a2e3ad21a8c9d40f3469 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 10:58:24 +0530 Subject: [PATCH 214/513] test linux tar.gz creation --- release-pipelines.yml | 954 +++++++++++++++++++++--------------------- 1 file changed, 477 insertions(+), 477 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7a4e8ac42..087ab71ba 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -310,389 +310,389 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_amd64 - value: $(work_dir)/linux_amd64 - - name: linux_se_amd64 - value: $(work_dir)/linux_se_amd64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_amd64) - mkdir $(linux_se_amd64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(linux_amd64)/ - sudo ls -lRt $(linux_se_amd64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | - cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_amd64 + # value: $(work_dir)/linux_amd64 + # - name: linux_se_amd64 + # value: $(work_dir)/linux_se_amd64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_amd64) + # mkdir $(linux_se_amd64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_amd64)/ + # sudo ls -lRt $(linux_se_amd64)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # - script: | + # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add binaries to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64)/ - cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=amd64 - export GOOS=linux - $(linux_amd64)/azcopy_linux_amd64 --version - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64)/ + # cp NOTICE.txt $(linux_se_amd64)/ + # displayName: 'Copy required files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # $(linux_amd64)/azcopy_linux_amd64 --version + # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # # Run azcopy --version and capture the output + # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_arm64 - value: $(work_dir)/linux_arm64 - - name: linux_se_arm64 - value: $(work_dir)/linux_se_arm64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_arm64) - mkdir $(linux_se_arm64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_arm64 + # value: $(work_dir)/linux_arm64 + # - name: linux_se_arm64 + # value: $(work_dir)/linux_se_arm64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_arm64) + # mkdir $(linux_se_arm64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(linux_arm64)/ - sudo ls -lRt $(linux_se_arm64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | - cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_arm64)/ + # sudo ls -lRt $(linux_se_arm64)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # - script: | + # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add binaries to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ - cp NOTICE.txt $(linux_se_arm64)/ - displayName: 'Copy required files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=arm64 - export GOOS=linux - $(linux_arm64)/azcopy_linux_arm64 --version - azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_se_arm64)/ + # displayName: 'Copy required files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # $(linux_arm64)/azcopy_linux_arm64 --version + # azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -787,114 +787,114 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: darwin_amd_dir - value: '$(work_dir)/azcopy_darwin_amd64' - - name: darwin_arm_dir - value: '$(work_dir)/azcopy_darwin_arm64' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - mkdir $(darwin_amd_dir) - mkdir $(darwin_arm_dir) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-11' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: darwin_amd_dir + # value: '$(work_dir)/azcopy_darwin_amd64' + # - name: darwin_arm_dir + # value: '$(work_dir)/azcopy_darwin_arm64' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # mkdir $(darwin_amd_dir) + # mkdir $(darwin_arm_dir) + # mkdir $(archives) + # displayName: 'Create required directories' + + # - script: | + # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - script: | - cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - displayName: "Copy binaries to staging directory" - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - displayName: "Remove binaries from staging directory" - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - displayName: 'Copy NOTICE file' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + # displayName: "Copy binaries to staging directory" + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # displayName: "Remove binaries from staging directory" + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # displayName: 'Copy NOTICE file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts From 532ad0e17909f8a3e5241debd1f1a17fddcdab68 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 11:00:32 +0530 Subject: [PATCH 215/513] test linux tar.gz creation --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 087ab71ba..eaaec6565 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -9,7 +9,7 @@ parameters: type: string default: 'azcopy-' - - name: test_mode + - name: test_mode displayName: 'Test Mode' type: boolean default: false From f6d23e1321f11de2d4bdcb6ddeb87004df55b0dc Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 11:01:38 +0530 Subject: [PATCH 216/513] test linux tar.gz creation --- release-pipelines.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index eaaec6565..dabfe37d1 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1750,25 +1750,25 @@ stages: scriptType: bash scriptLocation: inlineScript inlineScript: | - # Find the current version - raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) - azcopy_version=${raw_version_output[2]} - - today=`date +"%Y%m%d"` - if [ $(test_mode) = "on" ]; then - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - else - container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - fi - echo "Artifacts will be uploaded to: $container_url" - - executable_name=azcopy_linux_amd64 - - # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/*.zip" "$container_url" --recursive --put-md5=true + # Find the current version + raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) + azcopy_version=${raw_version_output[2]} + + today=`date +"%Y%m%d"` + if [ $(test_mode) = "on" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy_linux_amd64 + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/*.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From d3e08061d369281d98aec896631b4fd53c47ced0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 11:05:19 +0530 Subject: [PATCH 217/513] test linux tar.gz creation --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index dabfe37d1..f63cc1e0d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1751,7 +1751,7 @@ stages: scriptLocation: inlineScript inlineScript: | # Find the current version - raw_version_output=(`$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version`) + raw_version_output=($($(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version)) azcopy_version=${raw_version_output[2]} today=`date +"%Y%m%d"` From 44ad8602941179a40861182d32b10c7d0ec2e1b1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 11:56:04 +0530 Subject: [PATCH 218/513] test linux tar.gz creation --- release-pipelines.yml | 1141 ++++++++++++++++++++++++----------------- 1 file changed, 659 insertions(+), 482 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f63cc1e0d..963ba8581 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -310,389 +310,389 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_amd64 - # value: $(work_dir)/linux_amd64 - # - name: linux_se_amd64 - # value: $(work_dir)/linux_se_amd64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_amd64) - # mkdir $(linux_se_amd64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_amd64)/ - # sudo ls -lRt $(linux_se_amd64)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) - - # - script: | - # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add binaries to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + - script: | + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64)/ - # cp NOTICE.txt $(linux_se_amd64)/ - # displayName: 'Copy required files' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # $(linux_amd64)/azcopy_linux_amd64 --version - # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # # Run azcopy --version and capture the output - # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy_linux_amd64 --version + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_arm64 - # value: $(work_dir)/linux_arm64 - # - name: linux_se_arm64 - # value: $(work_dir)/linux_se_arm64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_arm64) - # mkdir $(linux_se_arm64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_arm64)/ - # sudo ls -lRt $(linux_se_arm64)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) - - # - script: | - # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add binaries to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + - script: | + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_se_arm64)/ - # displayName: 'Copy required files' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # $(linux_arm64)/azcopy_linux_arm64 --version - # azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + displayName: 'Copy required files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy_linux_arm64 --version + azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -787,114 +787,114 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-11' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: darwin_amd_dir - # value: '$(work_dir)/azcopy_darwin_amd64' - # - name: darwin_arm_dir - # value: '$(work_dir)/azcopy_darwin_arm64' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # mkdir $(darwin_amd_dir) - # mkdir $(darwin_arm_dir) - # mkdir $(archives) - # displayName: 'Create required directories' - - # - script: | - # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' - # - script: | - # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - # displayName: "Copy binaries to staging directory" - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # displayName: "Remove binaries from staging directory" - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # displayName: 'Copy NOTICE file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -953,8 +953,27 @@ stages: sudo apt install apt-transport-https -y sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - + # Send images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: 'azcopy_linux*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: 'ESRP CodeSigning azCopy linux' inputs: @@ -1064,9 +1083,9 @@ stages: displayName: 'List Artifacts' - script: | - mkdir $(windows_amd64) - mkdir $(windows_386) - mkdir $(archives) + mkdir -p $(windows_amd64) + mkdir -p $(windows_386) + mkdir -p $(archives) displayName: 'Create required directories' - script: | @@ -1225,7 +1244,7 @@ stages: AuthCertName: 'azcopy-auth' AuthSignCertName: 'azcopy-sign-test' FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' + Pattern: '*.zip' signConfigType: inlineSignParams inlineOperation: | [ @@ -1286,6 +1305,164 @@ stages: artifactName: 'azCopy-mac-signed' displayName: 'Publish Signed Artifacts' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + # Send linux images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: 'azcopy_linux*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows exectuable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: '*.exe' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + # SignArtifacts end here - stage: TestArtifacts From 4782679c278b00eba40f9b8208df982a4575589d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 12:21:20 +0530 Subject: [PATCH 219/513] test linux tar.gz creation --- release-pipelines.yml | 129 ++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 54 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 963ba8581..8d3a70609 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -752,29 +752,6 @@ stages: displayName: 'Publish binaries' condition: succeededOrFailed() - # - task: PowerShell@2 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # # Run azcopy --version and capture the output - # $azcopyOutput = & "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" --version - - # # Output the raw azcopy version output for debugging - # Write-Output "Raw azcopy output: $azcopyOutput" - - # # Example output: "azcopy version 10.25.0~Preview.1" or "azcopy version 10.25.0" - # # Extract the version part using string manipulation - # if ($azcopyOutput -match "azcopy version (\d+\.\d+\.\d+)(~Preview\.\d+)?") { - # $version = $matches[1] - # Write-Output "Extracted version: $version" - - # # Set the pipeline variable - # #Write-Host "##vso[task.setvariable variable=AZCOPY_VERSION;isOutput=true]$version" - # } else { - # Write-Error "Unable to extract version from azcopy output" - # } - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-windows-temp' @@ -954,26 +931,6 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - # Send images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: 'azcopy_linux*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: 'ESRP CodeSigning azCopy linux' inputs: @@ -1155,20 +1112,22 @@ stages: cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_386) + sudo ls -ltR $(windows_amd64) + sudo ls -ltR $(windows_386) displayName: 'Copy required files' - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip' + archiveFile: '$(archives)/azcopy_windows_amd64.zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip' + archiveFile: '$(archives)/azcopy_windows_386.zip' continueOnError: true - script: | @@ -1442,27 +1401,23 @@ stages: # Validate signed images have md5sum changed - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ displayName: 'Make Artifacts executable' - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-linux-signed' + artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Artifacts' - # SignArtifacts end here - stage: TestArtifacts @@ -1864,6 +1819,72 @@ stages: azcopy_linux_amd64 --help displayName: 'Check Help' + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Install zip and unzip utilities + brew install zip unzip + + # Define variables + ZIP_FILE_PATH=$(ls "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Change to the directory containing azcopy + cd $EXTRACT_DIR + + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + + # TestArtifacts ends here - stage: ReleaseArtifacts From c551f834b7ec6ab76ac0e8df67ddc231a04d7576 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 12:46:28 +0530 Subject: [PATCH 220/513] test linux tar.gz creation --- release-pipelines.yml | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8d3a70609..b73e96d43 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1328,32 +1328,6 @@ stages: } ] - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 displayName: 'Sign Windows exectuable' inputs: @@ -1399,6 +1373,32 @@ stages: ] SessionTimeout: 30 + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* From f0c45b46435fdfaba84d91b324fdcee337d5c299 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 13:10:40 +0530 Subject: [PATCH 221/513] test linux tar.gz creation --- release-pipelines.yml | 55 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b73e96d43..217265e5f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1114,6 +1114,7 @@ stages: cp NOTICE.txt $(windows_386) sudo ls -ltR $(windows_amd64) sudo ls -ltR $(windows_386) + suco chmod 777 $(archives)/ displayName: 'Copy required files' - task: ArchiveFiles@2 @@ -1305,7 +1306,7 @@ stages: wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y + sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" # Send linux images for signing @@ -1883,7 +1884,59 @@ stages: # Run the azcopy version command ./azcopy_darwin_amd64 --version displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + + - script: | + # Define variables + $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Run the azcopy version command + .\azcopy.exe --version + displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From cf46f7c082de0ad66c1631ee9cc0a10d393205fd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 13:11:34 +0530 Subject: [PATCH 222/513] test linux tar.gz creation --- release-pipelines.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 217265e5f..4b6b02f32 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1919,24 +1919,24 @@ stages: downloadPath: $(root_dir) itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - script: | - # Define variables - $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + - script: | + # Define variables + $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - # Extract the zip file - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - - # Change to the directory containing azcopy - Set-Location -Path $extractDir + # Extract the zip file + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + + # Change to the directory containing azcopy + Set-Location -Path $extractDir - # Run the azcopy version command - .\azcopy.exe --version - displayName: 'Extract Files and Run Version Command on Windows' + # Run the azcopy version command + .\azcopy.exe --version + displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From 36c3446aa3715f445e3c1be8ec54b3b74d48c38b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 13:12:22 +0530 Subject: [PATCH 223/513] test linux tar.gz creation --- release-pipelines.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4b6b02f32..e6695175d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1920,23 +1920,23 @@ stages: itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - script: | - # Define variables - $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + # Define variables + $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - # Extract the zip file - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - - # Change to the directory containing azcopy - Set-Location -Path $extractDir + # Extract the zip file + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + + # Change to the directory containing azcopy + Set-Location -Path $extractDir - # Run the azcopy version command - .\azcopy.exe --version - displayName: 'Extract Files and Run Version Command on Windows' + # Run the azcopy version command + .\azcopy.exe --version + displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From fc779536b898fd9949b1d40140fab186948a8287 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 13:42:16 +0530 Subject: [PATCH 224/513] test linux tar.gz creation --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e6695175d..645498b4f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1114,7 +1114,7 @@ stages: cp NOTICE.txt $(windows_386) sudo ls -ltR $(windows_amd64) sudo ls -ltR $(windows_386) - suco chmod 777 $(archives)/ + sudo chmod 777 $(archives)/ displayName: 'Copy required files' - task: ArchiveFiles@2 @@ -1306,7 +1306,7 @@ stages: wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y + sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" # Send linux images for signing From 7eddf41bb8238bb1d2615264310a36ea136cb7b0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 14:03:04 +0530 Subject: [PATCH 225/513] test linux tar.gz creation --- release-pipelines.yml | 53 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 645498b4f..f7db884c8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1114,7 +1114,8 @@ stages: cp NOTICE.txt $(windows_386) sudo ls -ltR $(windows_amd64) sudo ls -ltR $(windows_386) - sudo chmod 777 $(archives)/ + zip -r $(archives)/azcopy_windows_amd64.zip . + zip -r $(archives)/azcopy_windows_386.zip . displayName: 'Copy required files' - task: ArchiveFiles@2 @@ -1374,31 +1375,31 @@ stages: ] SessionTimeout: 30 - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Mac executable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", + # "OperationCode":"MacAppDeveloperSign", + # "Parameters":{ + # "Hardening":"--options=runtime" + # }, + # "ToolName":"sign", + # "ToolVersion":"1.0" + # } + # ] + # SessionTimeout: 30 # Validate signed images have md5sum changed - script: | From 6ca49e31601372e4a6d29c44692843e1b2ff25bf Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 15:15:18 +0530 Subject: [PATCH 226/513] test linux tar.gz creation --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index f7db884c8..794846d87 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1054,6 +1054,7 @@ stages: sudo dpkg -i packages-microsoft-prod.deb sudo apt install apt-transport-https -y sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip displayName: "Update dependencies" - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 From 5ebd65499857086dfe1ca9ade942806157a92983 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 15:55:35 +0530 Subject: [PATCH 227/513] test linux tar.gz creation --- release-pipelines.yml | 51 +++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 794846d87..dbae5531e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1113,30 +1113,28 @@ stages: cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_386) - sudo ls -ltR $(windows_amd64) - sudo ls -ltR $(windows_386) - zip -r $(archives)/azcopy_windows_amd64.zip . - zip -r $(archives)/azcopy_windows_386.zip . + zip -r $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip . + zip -r $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip . displayName: 'Copy required files' - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64.zip' + archiveFile: '$(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386.zip' + archiveFile: '$(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip' continueOnError: true - script: | cp $(archives)/azcopy_windows_amd64.zip $(Build.ArtifactStagingDirectory) cp $(archives)/azcopy_windows_386.zip $(Build.ArtifactStagingDirectory) - cp $(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - script: | @@ -1267,6 +1265,19 @@ stages: artifactName: 'azCopy-mac-signed' displayName: 'Publish Signed Artifacts' + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped + unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped + mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/unzipped + displayName: 'Copy signed mac build to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + - job: Set_4 timeoutInMinutes: 120 strategy: @@ -1376,32 +1387,6 @@ stages: ] SessionTimeout: 30 - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Mac executable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # Pattern: 'azcopy_darwin*' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - # "OperationCode":"MacAppDeveloperSign", - # "Parameters":{ - # "Hardening":"--options=runtime" - # }, - # "ToolName":"sign", - # "ToolVersion":"1.0" - # } - # ] - # SessionTimeout: 30 - # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* From f0d6d476b2d78805f820f09d49904f35e2826a49 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 16:15:04 +0530 Subject: [PATCH 228/513] test linux tar.gz creation --- release-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index dbae5531e..620eca81a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1134,7 +1134,6 @@ stages: - script: | cp $(archives)/azcopy_windows_amd64.zip $(Build.ArtifactStagingDirectory) cp $(archives)/azcopy_windows_386.zip $(Build.ArtifactStagingDirectory) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_v7_arm.exe $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - script: | From 3f2f578359a0f5b5b643555e36b223c308469d36 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 16:53:16 +0530 Subject: [PATCH 229/513] test linux tar.gz creation --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 620eca81a..fa1a2e182 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1132,8 +1132,8 @@ stages: continueOnError: true - script: | - cp $(archives)/azcopy_windows_amd64.zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386.zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - script: | From a527ca235f1c7ed9aa0f649e7a2e539d9673ee3c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 18:52:25 +0530 Subject: [PATCH 230/513] test linux tar.gz creation --- release-pipelines.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index fa1a2e182..6316e8a32 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1013,8 +1013,6 @@ stages: value: '$(work_dir)/azcopy_windows_386' - name: archives value: '$(work_dir)/archives' - - name: AZCOPY_VERSION - value: $[ dependencies.SetVariables.outputs['SetVars.AZCOPY_VERSION'] ] steps: - script: | @@ -1032,7 +1030,8 @@ stages: displayName: 'Download linux binary for extracting version' inputs: artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory)/azcopy_linux_amd64 + downloadPath: $(root_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) @@ -1108,27 +1107,38 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' + - script: | + # Run azcopy --version and capture the output + azcopyOutput=$($($(root_dir)/azCopy_binaries)/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + displayName: 'Extract the generated build version' + - script: | cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip . - zip -r $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip . + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . displayName: 'Copy required files' - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit Build' inputs: rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' continueOnError: true - script: | @@ -1137,6 +1147,7 @@ stages: displayName: 'Copy zip to staging directory' - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy_windows_temp sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* displayName: 'List Signed Artifacts' From ef9653e3cec1ad56539a48ee588a9730664e64d9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 19:13:19 +0530 Subject: [PATCH 231/513] test linux tar.gz creation --- release-pipelines.yml | 1752 +++++++++++++++++++++-------------------- 1 file changed, 878 insertions(+), 874 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6316e8a32..ca125232b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -310,389 +310,389 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_amd64 - value: $(work_dir)/linux_amd64 - - name: linux_se_amd64 - value: $(work_dir)/linux_se_amd64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_amd64) - mkdir $(linux_se_amd64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(linux_amd64)/ - sudo ls -lRt $(linux_se_amd64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | - cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_amd64 + # value: $(work_dir)/linux_amd64 + # - name: linux_se_amd64 + # value: $(work_dir)/linux_se_amd64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_amd64) + # mkdir $(linux_se_amd64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_amd64)/ + # sudo ls -lRt $(linux_se_amd64)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # - script: | + # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add binaries to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64)/ - cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=amd64 - export GOOS=linux - $(linux_amd64)/azcopy_linux_amd64 --version - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64)/ + # cp NOTICE.txt $(linux_se_amd64)/ + # displayName: 'Copy required files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # $(linux_amd64)/azcopy_linux_amd64 --version + # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # # Run azcopy --version and capture the output + # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_arm64 - value: $(work_dir)/linux_arm64 - - name: linux_se_arm64 - value: $(work_dir)/linux_se_arm64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_arm64) - mkdir $(linux_se_arm64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_arm64 + # value: $(work_dir)/linux_arm64 + # - name: linux_se_arm64 + # value: $(work_dir)/linux_se_arm64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_arm64) + # mkdir $(linux_se_arm64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(linux_arm64)/ - sudo ls -lRt $(linux_se_arm64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | - cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_arm64)/ + # sudo ls -lRt $(linux_se_arm64)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # - script: | + # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add binaries to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ - cp NOTICE.txt $(linux_se_arm64)/ - displayName: 'Copy required files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=arm64 - export GOOS=linux - $(linux_arm64)/azcopy_linux_arm64 --version - azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_se_arm64)/ + # displayName: 'Copy required files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # $(linux_arm64)/azcopy_linux_arm64 --version + # azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -729,267 +729,271 @@ stages: GOOS: windows CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: darwin_amd_dir - value: '$(work_dir)/azcopy_darwin_amd64' - - name: darwin_arm_dir - value: '$(work_dir)/azcopy_darwin_arm64' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - mkdir $(darwin_amd_dir) - mkdir $(darwin_arm_dir) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' - - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - script: | - cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - displayName: "Copy binaries to staging directory" - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - displayName: "Remove binaries from staging directory" - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - displayName: 'Copy NOTICE file' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here - - - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - Pattern: '*.rpm, *.deb, *tar.gz' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' + - task: PublishBuildArtifacts@1 inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # Validate signed images have md5sum changed - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - displayName: 'Make Artifacts executable' + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + displayName: 'Remove arm7 binary from staging directory' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-11' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: darwin_amd_dir + # value: '$(work_dir)/azcopy_darwin_amd64' + # - name: darwin_arm_dir + # value: '$(work_dir)/azcopy_darwin_arm64' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # mkdir $(darwin_amd_dir) + # mkdir $(darwin_arm_dir) + # mkdir $(archives) + # displayName: 'Create required directories' + + # - script: | + # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' + + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - script: | + # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + # displayName: "Copy binaries to staging directory" + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # displayName: "Remove binaries from staging directory" + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # displayName: 'Copy NOTICE file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() + + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here + + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none + + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy linux' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + # Pattern: '*.rpm, *.deb, *tar.gz' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -1030,7 +1034,7 @@ stages: displayName: 'Download linux binary for extracting version' inputs: artifactName: 'azCopy-binaries' - downloadPath: $(root_dir)/ + downloadPath: $(work_dir)/ itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - script: | @@ -1109,7 +1113,7 @@ stages: - script: | # Run azcopy --version and capture the output - azcopyOutput=$($($(root_dir)/azCopy_binaries)/azcopy_linux_amd64 --version) + azcopyOutput=$($(work_dir)/azCopy_binaries/azcopy_linux_amd64 --version) if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') @@ -1158,263 +1162,263 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: '*.zip' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Notarize Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppNotarize", - "Parameters":{ - "BundleId":"com.microsoft.AzCopy" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Mac executable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + # Pattern: '*.zip' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", + # "OperationCode":"MacAppDeveloperSign", + # "Parameters":{ + # "Hardening":"--options=runtime" + # }, + # "ToolName":"sign", + # "ToolVersion":"1.0" + # } + # ] + # SessionTimeout: 30 + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Notarize Mac executable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + # Pattern: 'azcopy_darwin*' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "KeyCode":"CP-401337-Apple", + # "OperationCode":"MacAppNotarize", + # "Parameters":{ + # "BundleId":"com.microsoft.AzCopy" + # }, + # "ToolName":"sign", + # "ToolVersion":"1.0" + # } + # ] + # SessionTimeout: 30 + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped - unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped - mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/unzipped - displayName: 'Copy signed mac build to staging directory' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped + # unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped + # mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped + # displayName: 'Copy signed mac build to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - # Send linux images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: 'azcopy_linux*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: '*.exe' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # # Send linux images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy linux' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # Pattern: 'azcopy_linux*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Windows exectuable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # Pattern: '*.exe' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", + # "operationSetCode": "SigntoolSign", + # "parameters": [ + # { + # "parameterName": "OpusName", + # "parameterValue": "Microsoft" + # }, + # { + # "parameterName": "OpusInfo", + # "parameterValue": "http://www.microsoft.com" + # }, + # { + # "parameterName": "PageHash", + # "parameterValue": "/NPH" + # }, + # { + # "parameterName": "FileDigest", + # "parameterValue": "/fd sha256" + # }, + # { + # "parameterName": "TimeStamp", + # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + # } + # ], + # "toolName": "signtool.exe", + # "toolVersion": "6.2.9304.0" + # } + # ] + # SessionTimeout: 30 + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here From f5b05f11b475c525f44fb8a09374d924d37d4d10 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 19:19:56 +0530 Subject: [PATCH 232/513] test linux tar.gz creation --- release-pipelines.yml | 1739 +++++++++++++++++++++-------------------- 1 file changed, 870 insertions(+), 869 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ca125232b..a4ac173a5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -310,389 +310,389 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_amd64 - # value: $(work_dir)/linux_amd64 - # - name: linux_se_amd64 - # value: $(work_dir)/linux_se_amd64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_amd64) - # mkdir $(linux_se_amd64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_amd64)/ - # sudo ls -lRt $(linux_se_amd64)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) - - # - script: | - # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add binaries to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + - script: | + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64)/ - # cp NOTICE.txt $(linux_se_amd64)/ - # displayName: 'Copy required files' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # $(linux_amd64)/azcopy_linux_amd64 --version - # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # # Run azcopy --version and capture the output - # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy_linux_amd64 --version + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_arm64 - # value: $(work_dir)/linux_arm64 - # - name: linux_se_arm64 - # value: $(work_dir)/linux_se_arm64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_arm64) - # mkdir $(linux_se_arm64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_arm64)/ - # sudo ls -lRt $(linux_se_arm64)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) - - # - script: | - # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add binaries to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) + + - script: | + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_se_arm64)/ - # displayName: 'Copy required files' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # $(linux_arm64)/azcopy_linux_arm64 --version - # azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + displayName: 'Copy required files' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy_linux_arm64 --version + azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -737,263 +737,263 @@ stages: GOOS: windows CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here + + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + Pattern: '*.rpm, *.deb, *tar.gz' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] - - task: PublishBuildArtifacts@1 + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + # Validate signed images have md5sum changed - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - displayName: 'Remove arm7 binary from staging directory' + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-11' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: darwin_amd_dir - # value: '$(work_dir)/azcopy_darwin_amd64' - # - name: darwin_arm_dir - # value: '$(work_dir)/azcopy_darwin_arm64' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # mkdir $(darwin_amd_dir) - # mkdir $(darwin_arm_dir) - # mkdir $(archives) - # displayName: 'Create required directories' - - # - script: | - # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' - - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - script: | - # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - # displayName: "Copy binaries to staging directory" - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # displayName: "Remove binaries from staging directory" - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # displayName: 'Copy NOTICE file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() - - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here - - - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none - - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy linux' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - # Pattern: '*.rpm, *.deb, *tar.gz' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -1162,263 +1162,264 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Mac executable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - # Pattern: '*.zip' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - # "OperationCode":"MacAppDeveloperSign", - # "Parameters":{ - # "Hardening":"--options=runtime" - # }, - # "ToolName":"sign", - # "ToolVersion":"1.0" - # } - # ] - # SessionTimeout: 30 - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Notarize Mac executable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - # Pattern: 'azcopy_darwin*' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "KeyCode":"CP-401337-Apple", - # "OperationCode":"MacAppNotarize", - # "Parameters":{ - # "BundleId":"com.microsoft.AzCopy" - # }, - # "ToolName":"sign", - # "ToolVersion":"1.0" - # } - # ] - # SessionTimeout: 30 - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + Pattern: '*.zip' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Notarize Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + Pattern: 'azcopy_darwin*' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"CP-401337-Apple", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped - # unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped - # mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped - # displayName: 'Copy signed mac build to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + rm-rf $(Build.ArtifactStagingDirectory)/*.zip + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped + unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped + mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/unzipped + displayName: 'Copy signed mac build to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # # Send linux images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy linux' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # Pattern: 'azcopy_linux*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Windows exectuable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # Pattern: '*.exe' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - # "operationSetCode": "SigntoolSign", - # "parameters": [ - # { - # "parameterName": "OpusName", - # "parameterValue": "Microsoft" - # }, - # { - # "parameterName": "OpusInfo", - # "parameterValue": "http://www.microsoft.com" - # }, - # { - # "parameterName": "PageHash", - # "parameterValue": "/NPH" - # }, - # { - # "parameterName": "FileDigest", - # "parameterValue": "/fd sha256" - # }, - # { - # "parameterName": "TimeStamp", - # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - # } - # ], - # "toolName": "signtool.exe", - # "toolVersion": "6.2.9304.0" - # } - # ] - # SessionTimeout: 30 - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + # Send linux images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: 'azcopy_linux*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows exectuable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: '*.exe' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here From 6564764cc7f2d06c308814b703c904b4b7ff12f9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 19:37:34 +0530 Subject: [PATCH 233/513] test linux tar.gz creation --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a4ac173a5..1aa44cdee 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1113,7 +1113,7 @@ stages: - script: | # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy_binaries/azcopy_linux_amd64 --version) + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') From ac970af2f69effbb98f80fb099c590a48efc5044 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 19:54:22 +0530 Subject: [PATCH 234/513] test linux tar.gz creation --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1aa44cdee..0590a7eee 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1112,6 +1112,7 @@ stages: displayName: 'Make Artifacts executable' - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 # Run azcopy --version and capture the output azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) From ccbd8a3270cc4bd161fcf0fecf52d9d862663f9d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 20:27:10 +0530 Subject: [PATCH 235/513] test linux tar.gz creation --- release-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0590a7eee..203643eac 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1121,6 +1121,8 @@ stages: if [[ -n "$version" ]]; then echo "Extracted version: $version" echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi displayName: 'Extract the generated build version' - script: | From bc0d0d259b830a50b1397e5fc9af2c2005333788 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 20:45:56 +0530 Subject: [PATCH 236/513] test linux tar.gz creation --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 203643eac..c9270e734 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1154,7 +1154,7 @@ stages: displayName: 'Copy zip to staging directory' - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy_windows_temp + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* displayName: 'List Signed Artifacts' From 00276b1eb0990dc88aa0940e98b241429e802c7c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 21:51:05 +0530 Subject: [PATCH 237/513] test linux tar.gz creation --- release-pipelines.yml | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c9270e734..c55949e30 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1870,11 +1870,11 @@ stages: eval "$(/opt/homebrew/bin/brew shellenv)" fi - # Install zip and unzip utilities + # # Install zip and unzip utilities brew install zip unzip # Define variables - ZIP_FILE_PATH=$(ls "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + ZIP_FILE_PATH=$(ls "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip") EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" # Create extraction directory @@ -1924,23 +1924,21 @@ stages: downloadPath: $(root_dir) itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - script: | - # Define variables - $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Run the azcopy version command - .\azcopy.exe --version + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + + $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + + Set-Location -Path $extractDir + .\azcopy.exe --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From b5377910fe7e90c0d92c87560d414d9a87fa85c7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 21:52:44 +0530 Subject: [PATCH 238/513] test win and mac artifacts --- release-pipelines.yml | 1974 ++++++++++++++++++++--------------------- 1 file changed, 987 insertions(+), 987 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c55949e30..eee94ab8d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -310,389 +310,389 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_amd64 - value: $(work_dir)/linux_amd64 - - name: linux_se_amd64 - value: $(work_dir)/linux_se_amd64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_amd64) - mkdir $(linux_se_amd64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(linux_amd64)/ - sudo ls -lRt $(linux_se_amd64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | - cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_amd64 + # value: $(work_dir)/linux_amd64 + # - name: linux_se_amd64 + # value: $(work_dir)/linux_se_amd64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_amd64) + # mkdir $(linux_se_amd64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_amd64)/ + # sudo ls -lRt $(linux_se_amd64)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # - script: | + # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add binaries to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64)/ - cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=amd64 - export GOOS=linux - $(linux_amd64)/azcopy_linux_amd64 --version - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64)/ + # cp NOTICE.txt $(linux_se_amd64)/ + # displayName: 'Copy required files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # $(linux_amd64)/azcopy_linux_amd64 --version + # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # # Run azcopy --version and capture the output + # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_arm64 - value: $(work_dir)/linux_arm64 - - name: linux_se_arm64 - value: $(work_dir)/linux_se_arm64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_arm64) - mkdir $(linux_se_arm64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_arm64 + # value: $(work_dir)/linux_arm64 + # - name: linux_se_arm64 + # value: $(work_dir)/linux_se_arm64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_arm64) + # mkdir $(linux_se_arm64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(linux_arm64)/ - sudo ls -lRt $(linux_se_arm64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | - cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_arm64)/ + # sudo ls -lRt $(linux_se_arm64)/ + # displayName: 'List the binaries' + # workingDirectory: $(root_dir) + + # - script: | + # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add binaries to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ - cp NOTICE.txt $(linux_se_arm64)/ - displayName: 'Copy required files' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=arm64 - export GOOS=linux - $(linux_arm64)/azcopy_linux_arm64 --version - azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_se_arm64)/ + # displayName: 'Copy required files' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # $(linux_arm64)/azcopy_linux_arm64 --version + # azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -881,119 +881,119 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - Pattern: '*.rpm, *.deb, *tar.gz' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy linux' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + # Pattern: '*.rpm, *.deb, *tar.gz' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy mariner' + # inputs: + # ConnectedServiceName: 'PMC ESRP azCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + # Pattern: '*.rpm, *.deb' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -1296,133 +1296,133 @@ stages: artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Mac Binary' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - # Send linux images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: 'azcopy_linux*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - Pattern: '*.exe' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # # Send linux images for signing + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + # displayName: 'ESRP CodeSigning azCopy linux' + # inputs: + # ConnectedServiceName: 'PMC ESRP AzCopy Signing' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # Pattern: 'azcopy_linux*' + # signConfigType: inlineSignParams + # VerboseLogin: true + # inlineOperation: | + # [ + # { + # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + # "OperationCode" : "LinuxSign", + # "Parameters" : {}, + # "ToolName" : "sign", + # "ToolVersion" : "1.0" + # } + # ] + + # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + # displayName: 'Sign Windows exectuable' + # inputs: + # ConnectedServiceName: 'ESRP KeyVault identity' + # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + # AuthAKVName: 'azcopy-esrp-kv' + # AuthCertName: 'azcopy-auth' + # AuthSignCertName: 'azcopy-sign-test' + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # Pattern: '*.exe' + # signConfigType: inlineSignParams + # inlineOperation: | + # [ + # { + # "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", + # "operationSetCode": "SigntoolSign", + # "parameters": [ + # { + # "parameterName": "OpusName", + # "parameterValue": "Microsoft" + # }, + # { + # "parameterName": "OpusInfo", + # "parameterValue": "http://www.microsoft.com" + # }, + # { + # "parameterName": "PageHash", + # "parameterValue": "/NPH" + # }, + # { + # "parameterName": "FileDigest", + # "parameterValue": "/fd sha256" + # }, + # { + # "parameterName": "TimeStamp", + # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + # } + # ], + # "toolName": "signtool.exe", + # "toolVersion": "6.2.9304.0" + # } + # ] + # SessionTimeout: 30 + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1430,400 +1430,400 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - #TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - checkout: none + + # #TODO: replace org name from dphulkar-msft to Azure + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*x86_64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-linux-signed/azcopy-*x86_64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - # TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # azcopy_linux_amd64 --version + # displayName: 'Check Version' + + # - script: | + # azcopy_linux_amd64 --help + # displayName: 'Check Help' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - checkout: none + + # # TODO: replace org name from dphulkar-msft to Azure + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*arm64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - azcopy_linux_arm64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_arm64 --help - displayName: 'Check Help' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-linux-signed/azcopy-*arm64.deb + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # azcopy_linux_arm64 --version + # displayName: 'Check Version' + + # - script: | + # azcopy_linux_arm64 --help + # displayName: 'Check Help' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - checkout: none + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' - #TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + # #TODO: replace org name from dphulkar-msft to Azure + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - azcopy_linux_arm64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_arm64 --help - displayName: 'Check Help' + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # azcopy_linux_arm64 --version + # displayName: 'Check Version' + + # - script: | + # azcopy_linux_arm64 --help + # displayName: 'Check Help' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - #TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' - - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install Git' - - #TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - - - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - checkout: none + + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # #TODO: replace org name from dphulkar-msft to Azure + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed + + # - script: | + # azcopy_linux_amd64 --version + # displayName: 'Check Version' + + # - script: | + # azcopy_linux_amd64 --help + # displayName: 'Check Help' + + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - checkout: none + + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install Git' + + # #TODO: replace org name from dphulkar-msft to Azure + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(root_dir)/azure-storage-azcopy + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "GoTool Custom Setup" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm + + # - script: | + # ls -l + # result=$(ls -1 | wc -l) + # if [ $result -ne 1 ]; then + # exit 1 + # fi + # displayName: 'List Downloaded Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + + # - script: | + # sudo rpm -qip azcopy*x86_64.rpm + # sudo rpm -i azcopy*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + + # - script: | + # azcopy_linux_amd64 --version + # displayName: 'Check Version' + + # - script: | + # azcopy_linux_amd64 --help + # displayName: 'Check Help' - job: Set_6 timeoutInMinutes: 360 From 68b34bb5ad06844d2e7e6b958dd9d771ce5ec5fb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 22:33:11 +0530 Subject: [PATCH 239/513] test win and mac artifacts --- release-pipelines.yml | 222 ++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 97 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index eee94ab8d..152f86399 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1288,6 +1288,7 @@ stages: unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ rm -rf $(Build.ArtifactStagingDirectory)/unzipped + sudo ls -ltR rm-rf $(Build.ArtifactStagingDirectory) displayName: 'Copy signed mac build to staging directory' # Push signed images to artifact directory @@ -1825,121 +1826,148 @@ stages: # azcopy_linux_amd64 --help # displayName: 'Check Help' - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-11' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi - # # Install zip and unzip utilities - brew install zip unzip + # # # Install zip and unzip utilities + # brew install zip unzip - # Define variables - ZIP_FILE_PATH=$(ls "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip") - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Create extraction directory - mkdir -p $EXTRACT_DIR + # # Create extraction directory + # mkdir -p $EXTRACT_DIR - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Change to the directory containing azcopy - cd $EXTRACT_DIR + # # Change to the directory containing azcopy + # cd $EXTRACT_DIR - # Run the azcopy version command - ./azcopy_darwin_amd64 --version - displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # # Run the azcopy version command + # ./azcopy_darwin_amd64 --version + # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(root_dir) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - - $zipFilePath = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed\azcopy.zip" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - - Set-Location -Path $extractDir - .\azcopy.exe --version - displayName: 'Extract Files and Run Version Command on Windows' + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } + + # $zipFilePath = $zipFile.FullName + + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } + + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir + + # # Verify that azcopy.exe exists + # if (-Not (Test-Path -Path ".\azcopy.exe")) { + # Write-Error "azcopy.exe not found in extraction directory: $extractDir" + # exit 1 + # } + + # # Run the azcopy version command + # .\azcopy.exe --version + # displayName: 'Extract Files and Run Version Command on Windows' + # TestArtifacts ends here From d12eaa36d7fb9314a32c1d5b6b161475b6f01c93 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 22:42:45 +0530 Subject: [PATCH 240/513] test win and mac artifacts --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 152f86399..82c4b8b3e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1427,10 +1427,10 @@ stages: # SignArtifacts end here - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: # - job: Set_1 # timeoutInMinutes: 120 # strategy: From fa7c2d56270fc3b5281145be0bbbbc3a4e694e02 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 22:43:48 +0530 Subject: [PATCH 241/513] test win and mac artifacts --- release-pipelines.yml | 130 +++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 82c4b8b3e..39d8775d6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1427,79 +1427,79 @@ stages: # SignArtifacts end here - # - stage: TestArtifacts - # dependsOn: SignArtifacts - # condition: succeeded('SignArtifacts') - # jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*x86_64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*x86_64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed - - # - script: | - # azcopy_linux_amd64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_amd64 --help - # displayName: 'Check Help' + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(root_dir)/azCopy-linux-signed + + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' # - job: Set_2 # timeoutInMinutes: 120 From c56bae8ce5b77e0cc6940ed96c91ff123a5af1a2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 22:55:40 +0530 Subject: [PATCH 242/513] test win and mac artifacts --- release-pipelines.yml | 1112 ++++++++++++++++++++--------------------- 1 file changed, 556 insertions(+), 556 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 39d8775d6..b30581f75 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -310,389 +310,389 @@ stages: #dependsOn: RunTests #condition: succeeded('RunTests') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_amd64 - # value: $(work_dir)/linux_amd64 - # - name: linux_se_amd64 - # value: $(work_dir)/linux_se_amd64 + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # mkdir $(linux_amd64) - # mkdir $(linux_se_amd64) - # mkdir $(archives) - # displayName: "Create required directories" + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - # - script: | - # sudo ls -lRt $(linux_amd64)/ - # sudo ls -lRt $(linux_se_amd64)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # - script: | - # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add binaries to staging directory' + - script: | + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64)/ - # cp NOTICE.txt $(linux_se_amd64)/ - # displayName: 'Copy required files' + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files' - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # $(linux_amd64)/azcopy_linux_amd64 --version - # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # # Run azcopy --version and capture the output - # azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy_linux_amd64 --version + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_arm64 - # value: $(work_dir)/linux_arm64 - # - name: linux_se_arm64 - # value: $(work_dir)/linux_se_arm64 + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # mkdir $(linux_arm64) - # mkdir $(linux_se_arm64) - # mkdir $(archives) - # displayName: "Create required directories" + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' - # - script: | - # sudo ls -lRt $(linux_arm64)/ - # sudo ls -lRt $(linux_se_arm64)/ - # displayName: 'List the binaries' - # workingDirectory: $(root_dir) + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + displayName: 'List the binaries' + workingDirectory: $(root_dir) - # - script: | - # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add binaries to staging directory' + - script: | + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add binaries to staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_se_arm64)/ - # displayName: 'Copy required files' + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + displayName: 'Copy required files' - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # $(linux_arm64)/azcopy_linux_arm64 --version - # azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy_linux_arm64 --version + azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -881,119 +881,119 @@ stages: dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy linux' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - # Pattern: '*.rpm, *.deb, *tar.gz' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy mariner' - # inputs: - # ConnectedServiceName: 'PMC ESRP azCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - # Pattern: '*.rpm, *.deb' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + Pattern: '*.rpm, *.deb, *tar.gz' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy mariner' + inputs: + ConnectedServiceName: 'PMC ESRP azCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + Pattern: '*.rpm, *.deb' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -1297,133 +1297,133 @@ stages: artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Mac Binary' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # # Send linux images for signing - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - # displayName: 'ESRP CodeSigning azCopy linux' - # inputs: - # ConnectedServiceName: 'PMC ESRP AzCopy Signing' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # Pattern: 'azcopy_linux*' - # signConfigType: inlineSignParams - # VerboseLogin: true - # inlineOperation: | - # [ - # { - # "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - # "OperationCode" : "LinuxSign", - # "Parameters" : {}, - # "ToolName" : "sign", - # "ToolVersion" : "1.0" - # } - # ] - - # - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - # displayName: 'Sign Windows exectuable' - # inputs: - # ConnectedServiceName: 'ESRP KeyVault identity' - # AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - # AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - # AuthAKVName: 'azcopy-esrp-kv' - # AuthCertName: 'azcopy-auth' - # AuthSignCertName: 'azcopy-sign-test' - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # Pattern: '*.exe' - # signConfigType: inlineSignParams - # inlineOperation: | - # [ - # { - # "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - # "operationSetCode": "SigntoolSign", - # "parameters": [ - # { - # "parameterName": "OpusName", - # "parameterValue": "Microsoft" - # }, - # { - # "parameterName": "OpusInfo", - # "parameterValue": "http://www.microsoft.com" - # }, - # { - # "parameterName": "PageHash", - # "parameterValue": "/NPH" - # }, - # { - # "parameterName": "FileDigest", - # "parameterValue": "/fd sha256" - # }, - # { - # "parameterName": "TimeStamp", - # "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - # } - # ], - # "toolName": "signtool.exe", - # "toolVersion": "6.2.9304.0" - # } - # ] - # SessionTimeout: 30 - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Send linux images for signing + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'ESRP CodeSigning azCopy linux' + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: 'azcopy_linux*' + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows exectuable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + Pattern: '*.exe' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here From ae24bb9ed21c012800e183450917f2d8fe36510b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 14 Jun 2024 23:51:35 +0530 Subject: [PATCH 243/513] test win and mac artifacts --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b30581f75..1c286d7cc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1283,12 +1283,12 @@ stages: displayName: 'Publish Signed Artifacts' - script: | - rm-rf $(Build.ArtifactStagingDirectory)/*.zip + rm -rf $(Build.ArtifactStagingDirectory)/*.zip mkdir -p $(Build.ArtifactStagingDirectory)/unzipped unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ rm -rf $(Build.ArtifactStagingDirectory)/unzipped - sudo ls -ltR rm-rf $(Build.ArtifactStagingDirectory) + sudo ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Copy signed mac build to staging directory' # Push signed images to artifact directory From 7919b4d151ab21e442f606781e4ae7dcd06a7b06 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sat, 15 Jun 2024 09:08:36 +0530 Subject: [PATCH 244/513] test public container release --- release-pipelines.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1c286d7cc..933c41e74 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2018,12 +2018,26 @@ stages: artifactName: 'azCopy-binaries' downloadPath: $(Build.ArtifactStagingDirectory) + - script: | + zip -r $(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip . + displayName: 'Create drop.zip file' + - task: ArchiveFiles@2 displayName: 'Archive all binaries to drop.zip' inputs: rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' archiveFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip' continueOnError: true + + - task: UsePythonVersion@0 + displayName: 'Use Python version' + inputs: + versionSpec: '3.x' + addToPath: true + + - script: | + pip install azure-cli + displayName: 'Install Azure CLI' - task: AzureCLI@2 displayName: 'Azure CLI' From ee3e430a782e9db31ffd4dde21e6de19beb2943c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sat, 15 Jun 2024 11:20:07 +0530 Subject: [PATCH 245/513] test public container release --- release-pipelines.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 933c41e74..463e280bd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2019,6 +2019,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + sudo apt-get install -y zip zip -r $(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip . displayName: 'Create drop.zip file' @@ -2124,20 +2125,13 @@ stages: - ${{ if eq(parameters.post_release, true) }}: - - script: | - mv azCopy-linux-signed/azcopy_linux_se_amd64 . - mv azCopy-linux-signed/azcopy_linux_se_arm64 . - mv azCopy-windows-signed/azcopy_windows_v7_arm.exe . - displayName: 'Move binaries from signed folders' - workingDirectory: $(Build.ArtifactStagingDirectory) - # add release tags & push to github #TODO: update github connection and repo name while merging to main - task: GithubRelease@1 inputs: githubConnection: 'dphulkar-msft-github-rel' repositoryName: 'dphulkar-msft/azure-storage-azcopy' - action: 'create' + action: 'edit' target: '$(Build.SourceVersion)' tagSource: 'userSpecifiedTag' @@ -2155,13 +2149,6 @@ stages: isPreRelease: ${{ parameters.prerelease }} assetUploadMode: replace - - script: | - mv azcopy_linux_se_amd64 azCopy-linux-signed/ - mv azcopy_linux_se_arm64 azCopy-linux-signed/ - # mv azcopy_windows_v7_arm.exe azCopy-windows-signed/ - displayName: 'Move binaries from signed folders' - workingDirectory: $(Build.ArtifactStagingDirectory) - - ${{ if eq(parameters.publish_artifacts, true) }}: - stage: PublishArtifacts dependsOn: ReleaseArtifacts From f0eabab3ad4666ca917a0117a18789c6fec4bf89 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sat, 15 Jun 2024 11:20:52 +0530 Subject: [PATCH 246/513] test public container release --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 463e280bd..fbb98a340 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2019,7 +2019,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - sudo apt-get install -y zip + sudo apt-get install -y zip zip -r $(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip . displayName: 'Create drop.zip file' From 2ff753edb20c2bc50c8ea18ba50e8215aaddcc0f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 16 Jun 2024 10:37:06 +0530 Subject: [PATCH 247/513] test public container release --- release-pipelines.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index fbb98a340..1526ff7d0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2030,12 +2030,29 @@ stages: archiveFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip' continueOnError: true + - task: Bash@3 + displayName: 'Install Python 3.x' + inputs: + targetType: 'inline' + script: | + sudo apt-get update + sudo apt-get install -y python3 python3-pip + - task: UsePythonVersion@0 displayName: 'Use Python version' inputs: versionSpec: '3.x' addToPath: true + - task: Bash@3 + displayName: 'Verify Python installation' + inputs: + targetType: 'inline' + script: | + python3 --version + pip3 --version + + - script: | pip install azure-cli displayName: 'Install Azure CLI' From ac1049c0626516006cd43584684ab998d7662f00 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 16 Jun 2024 11:54:08 +0530 Subject: [PATCH 248/513] test public container release --- release-pipelines.yml | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1526ff7d0..494ccb95c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2030,32 +2030,38 @@ stages: archiveFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip' continueOnError: true - - task: Bash@3 - displayName: 'Install Python 3.x' - inputs: - targetType: 'inline' - script: | - sudo apt-get update - sudo apt-get install -y python3 python3-pip - - - task: UsePythonVersion@0 - displayName: 'Use Python version' - inputs: - versionSpec: '3.x' - addToPath: true + # - task: Bash@3 + # displayName: 'Install Python 3.x' + # inputs: + # targetType: 'inline' + # script: | + # sudo apt-get update + # sudo apt-get install -y python3 python3-pip + + # - task: UsePythonVersion@0 + # displayName: 'Use Python version' + # inputs: + # versionSpec: '3.x' + # addToPath: true + + # - task: Bash@3 + # displayName: 'Verify Python installation' + # inputs: + # targetType: 'inline' + # script: | + # python3 --version + # pip3 --version + + # - script: | + # pip install azure-cli + # displayName: 'Install Azure CLI' - task: Bash@3 - displayName: 'Verify Python installation' + displayName: 'Install Azure CLI' inputs: targetType: 'inline' script: | - python3 --version - pip3 --version - - - - script: | - pip install azure-cli - displayName: 'Install Azure CLI' + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - task: AzureCLI@2 displayName: 'Azure CLI' From bf42340436dae865dc93f23bdcdef1a151477097 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 16 Jun 2024 23:23:50 +0530 Subject: [PATCH 249/513] test public container release --- release-pipelines.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 494ccb95c..f744f5945 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2015,19 +2015,19 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Mac Build Artifacts' inputs: - artifactName: 'azCopy-binaries' + artifactName: 'azCopy-signed-binaries' downloadPath: $(Build.ArtifactStagingDirectory) - script: | sudo apt-get install -y zip - zip -r $(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip . + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . displayName: 'Create drop.zip file' - task: ArchiveFiles@2 displayName: 'Archive all binaries to drop.zip' inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - archiveFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries/drop.zip' + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-signed-binaries' + archiveFile: '$(Build.ArtifactStagingDirectory)//drop.zip' continueOnError: true # - task: Bash@3 @@ -2071,11 +2071,12 @@ stages: scriptLocation: inlineScript inlineScript: | # Find the current version - raw_version_output=($($(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version)) + raw_version_output=($($(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/azcopy_linux_amd64 --version)) azcopy_version=${raw_version_output[2]} today=`date +"%Y%m%d"` - if [ $(test_mode) = "on" ]; then + + if [ "$(parameters.test_mode)" = "on" ]; then container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" else container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" @@ -2088,7 +2089,7 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/*.zip" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From f900c16ef2bcc46315c472b49027da9c9ce46a1c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 12:19:02 +0530 Subject: [PATCH 250/513] test public container release --- release-pipelines.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f744f5945..490aedb08 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1991,6 +1991,8 @@ stages: value: 'azcopyprivatedrops' - name: Release_Account_Name value: 'azcopyvnextrelease' + - name: test_mode + value: ${{ parameters.test_mode }} steps: # download artifacts that need to be published @@ -2076,8 +2078,8 @@ stages: today=`date +"%Y%m%d"` - if [ "$(parameters.test_mode)" = "on" ]; then - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + if [ "$test_mode" = "true" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" else container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" fi @@ -2086,10 +2088,10 @@ stages: executable_name=azcopy_linux_amd64 # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From ec830b3c48cff0042b3f2d6545efdb0745228c53 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 12:28:53 +0530 Subject: [PATCH 251/513] test public container release --- release-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 490aedb08..981cf878d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2015,9 +2015,9 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' + displayName: 'Download Binaries Build Artifacts' inputs: - artifactName: 'azCopy-signed-binaries' + artifactName: 'azCopy-binaries-signed' downloadPath: $(Build.ArtifactStagingDirectory) - script: | @@ -2028,7 +2028,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive all binaries to drop.zip' inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-signed-binaries' + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' archiveFile: '$(Build.ArtifactStagingDirectory)//drop.zip' continueOnError: true @@ -2073,7 +2073,7 @@ stages: scriptLocation: inlineScript inlineScript: | # Find the current version - raw_version_output=($($(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/azcopy_linux_amd64 --version)) + raw_version_output=($($(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version)) azcopy_version=${raw_version_output[2]} today=`date +"%Y%m%d"` @@ -2088,10 +2088,10 @@ stages: executable_name=azcopy_linux_amd64 # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-signed-binaries/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From 5146a66a8a1934c9f0033a2e173b8b7d6e23ca3e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 13:30:25 +0530 Subject: [PATCH 252/513] test public container release --- release-pipelines.yml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 981cf878d..30eda7cb3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -745,6 +745,14 @@ stages: GOARM: 7 GOOS: windows CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 - task: PublishBuildArtifacts@1 inputs: @@ -1013,6 +1021,8 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - name: windows_amd64 value: '$(work_dir)/azcopy_windows_amd64' + - name: windows_arm64 + value: '$(work_dir)/azcopy_windows_arm64' - name: windows_386 value: '$(work_dir)/azcopy_windows_386' - name: archives @@ -1044,6 +1054,7 @@ stages: - script: | mkdir -p $(windows_amd64) + mkdir -p $(windows_arm64) mkdir -p $(windows_386) mkdir -p $(archives) displayName: 'Create required directories' @@ -1127,20 +1138,30 @@ stages: - script: | cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64) cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . displayName: 'Copy required files' - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit Build' + displayName: 'Archive Windows 64 bit AMD Build' inputs: rootFolderOrFile: '$(windows_amd64)' archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: @@ -1150,6 +1171,7 @@ stages: - script: | cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' @@ -2031,6 +2053,12 @@ stages: rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' archiveFile: '$(Build.ArtifactStagingDirectory)//drop.zip' continueOnError: true + + - script: | + sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-signed + sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-signed # - task: Bash@3 # displayName: 'Install Python 3.x' From 5eeb7ed77ce92e5a60ad4654173c036fd89d14c2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 13:47:27 +0530 Subject: [PATCH 253/513] test public container release --- release-pipelines.yml | 259 +++++++++++++++++++----------------------- 1 file changed, 116 insertions(+), 143 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 30eda7cb3..9aa8921fe 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1848,148 +1848,146 @@ stages: # azcopy_linux_amd64 --help # displayName: 'Check Help' - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-11' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi - # # # Install zip and unzip utilities - # brew install zip unzip + # # Install zip and unzip utilities + # brew install zip unzip - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # # Create extraction directory - # mkdir -p $EXTRACT_DIR + # Create extraction directory + mkdir -p $EXTRACT_DIR - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # # Change to the directory containing azcopy - # cd $EXTRACT_DIR + # Change to the directory containing azcopy + cd $EXTRACT_DIR - # # Run the azcopy version command - # ./azcopy_darwin_amd64 --version - # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - # - job: Set_7 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - # $zipFilePath = $zipFile.FullName + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + $zipFilePath = $zipFile.FullName - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } - # # Verify that azcopy.exe exists - # if (-Not (Test-Path -Path ".\azcopy.exe")) { - # Write-Error "azcopy.exe not found in extraction directory: $extractDir" - # exit 1 - # } + # Change to the directory containing azcopy + Set-Location -Path $extractDir - # # Run the azcopy version command - # .\azcopy.exe --version - # displayName: 'Extract Files and Run Version Command on Windows' + # Verify that azcopy.exe exists + if (-Not (Test-Path -Path ".\azcopy.exe")) { + Write-Error "azcopy.exe not found in extraction directory: $extractDir" + exit 1 + } + # Run the azcopy version command + .\azcopy.exe --version + displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -2059,32 +2057,7 @@ stages: sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-signed sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-signed sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-signed - - # - task: Bash@3 - # displayName: 'Install Python 3.x' - # inputs: - # targetType: 'inline' - # script: | - # sudo apt-get update - # sudo apt-get install -y python3 python3-pip - - # - task: UsePythonVersion@0 - # displayName: 'Use Python version' - # inputs: - # versionSpec: '3.x' - # addToPath: true - - # - task: Bash@3 - # displayName: 'Verify Python installation' - # inputs: - # targetType: 'inline' - # script: | - # python3 --version - # pip3 --version - - # - script: | - # pip install azure-cli - # displayName: 'Install Azure CLI' + displayName: 'Provide required permissions' - task: Bash@3 displayName: 'Install Azure CLI' From 5853f98e0216dde2c6e9f35f73144041b26321d7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 15:21:42 +0530 Subject: [PATCH 254/513] test win and mac test --- release-pipelines.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9aa8921fe..21f084353 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1881,7 +1881,7 @@ stages: displayName: 'Download Build Artifacts' inputs: artifactName: 'azCopy-mac-signed' - downloadPath: $(root_dir) + downloadPath: $(Build.ArtifactStagingDirectory) itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - script: | @@ -1944,7 +1944,7 @@ stages: displayName: 'Download Build Artifacts' inputs: artifactName: 'azCopy-windows-signed' - downloadPath: $(root_dir) + downloadPath: $(Build.ArtifactStagingDirectory) itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - task: PowerShell@2 @@ -2052,13 +2052,6 @@ stages: archiveFile: '$(Build.ArtifactStagingDirectory)//drop.zip' continueOnError: true - - script: | - sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed - sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-signed - sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - sudo chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-signed - displayName: 'Provide required permissions' - - task: Bash@3 displayName: 'Install Azure CLI' inputs: @@ -2073,11 +2066,14 @@ stages: scriptType: bash scriptLocation: inlineScript inlineScript: | + # Ensure azcopy binary is executable + chmod +x $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 + # Find the current version - raw_version_output=($($(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version)) - azcopy_version=${raw_version_output[2]} + raw_version_output=$($(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') - today=`date +"%Y%m%d"` + today=$(date +"%Y%m%d") if [ "$test_mode" = "true" ]; then container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" From 7c55d3eedecf33e41d9f0da5e4941f5e1ce2321d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 15:23:32 +0530 Subject: [PATCH 255/513] test release to container --- release-pipelines.yml | 244 +++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 21f084353..ee65be99c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1848,146 +1848,146 @@ stages: # azcopy_linux_amd64 --help # displayName: 'Check Help' - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-11' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi - # # Install zip and unzip utilities - # brew install zip unzip + # # # Install zip and unzip utilities + # # brew install zip unzip - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Create extraction directory - mkdir -p $EXTRACT_DIR + # # Create extraction directory + # mkdir -p $EXTRACT_DIR - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Change to the directory containing azcopy - cd $EXTRACT_DIR + # # Change to the directory containing azcopy + # cd $EXTRACT_DIR - # Run the azcopy version command - ./azcopy_darwin_amd64 --version - displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # # Run the azcopy version command + # ./azcopy_darwin_amd64 --version + # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy.exe")) { - Write-Error "azcopy.exe not found in extraction directory: $extractDir" - exit 1 - } - - # Run the azcopy version command - .\azcopy.exe --version - displayName: 'Extract Files and Run Version Command on Windows' + # # Find the zip file matching the pattern + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } + + # $zipFilePath = $zipFile.FullName + + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } + + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir + + # # Verify that azcopy.exe exists + # if (-Not (Test-Path -Path ".\azcopy.exe")) { + # Write-Error "azcopy.exe not found in extraction directory: $extractDir" + # exit 1 + # } + + # # Run the azcopy version command + # .\azcopy.exe --version + # displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From b5b2ef41edd39ec3ab47f42981ac2f7f2b9ded86 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 16:32:23 +0530 Subject: [PATCH 256/513] test mac signing --- release-pipelines.yml | 244 +++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ee65be99c..90625e9f8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1848,146 +1848,146 @@ stages: # azcopy_linux_amd64 --help # displayName: 'Check Help' - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-11' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-11' + type: 'mac-os' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi - # # # Install zip and unzip utilities - # # brew install zip unzip + # # Install zip and unzip utilities + # brew install zip unzip - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # # Create extraction directory - # mkdir -p $EXTRACT_DIR + # Create extraction directory + mkdir -p $EXTRACT_DIR - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # # Change to the directory containing azcopy - # cd $EXTRACT_DIR + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 - # # Run the azcopy version command - # ./azcopy_darwin_amd64 --version - # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - # - job: Set_7 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # # Find the zip file matching the pattern - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } - - # $zipFilePath = $zipFile.FullName - - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } - - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir - - # # Verify that azcopy.exe exists - # if (-Not (Test-Path -Path ".\azcopy.exe")) { - # Write-Error "azcopy.exe not found in extraction directory: $extractDir" - # exit 1 - # } - - # # Run the azcopy version command - # .\azcopy.exe --version - # displayName: 'Extract Files and Run Version Command on Windows' + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Verify that azcopy.exe exists + if (-Not (Test-Path -Path ".\azcopy_windows_amd64.exe")) { + Write-Error "azcopy.exe not found in extraction directory: $extractDir" + exit 1 + } + + # Run the azcopy version command + .\azcopy.exe --version + displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From c2913e1eb70b8b5f7feedcbc5721e2f54e5d3069 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 16:35:49 +0530 Subject: [PATCH 257/513] test mac and win --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 90625e9f8..b66e5628a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1980,13 +1980,13 @@ stages: Set-Location -Path $extractDir # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64.exe")) { + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { Write-Error "azcopy.exe not found in extraction directory: $extractDir" exit 1 } # Run the azcopy version command - .\azcopy.exe --version + .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From 12b97ac13cc8c2e6f656dd5a13e76182a1459896 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 16:46:33 +0530 Subject: [PATCH 258/513] test release --- release-pipelines.yml | 244 ++++++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 117 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b66e5628a..8a99d3cfd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1848,146 +1848,146 @@ stages: # azcopy_linux_amd64 --help # displayName: 'Check Help' - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-11' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-11' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi - # # Install zip and unzip utilities - # brew install zip unzip + # # # Install zip and unzip utilities + # # brew install zip unzip - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Create extraction directory - mkdir -p $EXTRACT_DIR + # # Create extraction directory + # mkdir -p $EXTRACT_DIR - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Change to the directory containing azcopy - cd $EXTRACT_DIR/azcopy_darwin_amd64 + # # Change to the directory containing azcopy + # cd $EXTRACT_DIR/azcopy_darwin_amd64 - # Run the azcopy version command - ./azcopy_darwin_amd64 --version - displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # # Run the azcopy version command + # ./azcopy_darwin_amd64 --version + # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + # # Find the zip file matching the pattern + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } - $zipFilePath = $zipFile.FullName + # $zipFilePath = $zipFile.FullName - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } - # Change to the directory containing azcopy - Set-Location -Path $extractDir + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir - # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { - Write-Error "azcopy.exe not found in extraction directory: $extractDir" - exit 1 - } + # # Verify that azcopy.exe exists + # if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { + # Write-Error "azcopy.exe not found in extraction directory: $extractDir" + # exit 1 + # } - # Run the azcopy version command - .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version - displayName: 'Extract Files and Run Version Command on Windows' + # # Run the azcopy version command + # .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version + # displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -2015,7 +2015,7 @@ stages: value: ${{ parameters.test_mode }} steps: - # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 displayName: 'Download Linux Build Artifacts' inputs: @@ -2066,8 +2066,18 @@ stages: scriptType: bash scriptLocation: inlineScript inlineScript: | + # Define variables + artifactDir=$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + azcopyPath="$artifactDir/azcopy_linux_amd64" + # Ensure azcopy binary is executable - chmod +x $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 + chmod +x $azcopyPath + + # Verify the azcopy binary + if ! $azcopyPath --version; then + echo "Failed to verify azcopy binary." + exit 1 + fi # Find the current version raw_version_output=$($(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version) From 2a228e274f22568f288f932970b004a923d909f9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 18:15:03 +0530 Subject: [PATCH 259/513] test release --- release-pipelines.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8a99d3cfd..94d6015a5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2059,6 +2059,11 @@ stages: script: | curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + - script: | + cp $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 . + displayName: 'Copy executable binary to different location' + workingDirectory: $(Build.ArtifactStagingDirectory) + - task: AzureCLI@2 displayName: 'Azure CLI' inputs: @@ -2067,7 +2072,7 @@ stages: scriptLocation: inlineScript inlineScript: | # Define variables - artifactDir=$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + artifactDir=$(Build.ArtifactStagingDirectory) azcopyPath="$artifactDir/azcopy_linux_amd64" # Ensure azcopy binary is executable @@ -2078,9 +2083,9 @@ stages: echo "Failed to verify azcopy binary." exit 1 fi - + # Find the current version - raw_version_output=$($(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version) + raw_version_output=$($(Build.ArtifactStagingDirectory)/azcopy_linux_amd64 --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') today=$(date +"%Y%m%d") @@ -2095,10 +2100,10 @@ stages: executable_name=azcopy_linux_amd64 # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From 3397332a5bcc942d899e16eba029a6ae14f8e9e7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 18:28:00 +0530 Subject: [PATCH 260/513] test release --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 94d6015a5..5c3165a6d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -781,7 +781,7 @@ stages: strategy: matrix: MacOS: - imageName: 'macos-11' + imageName: 'macos-13' type: 'mac-os' pool: vmImage: $(imageName) @@ -1853,7 +1853,7 @@ stages: # strategy: # matrix: # MacOS: - # imageName: 'macos-11' + # imageName: 'macos-13' # type: 'mac-os' # pool: # vmImage: $(imageName) From a79ca8b0416bc804a0b2127e050aebdb8a4f7481 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 18:55:39 +0530 Subject: [PATCH 261/513] test release --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5c3165a6d..0358a965b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2043,6 +2043,7 @@ stages: - script: | sudo apt-get install -y zip zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version displayName: 'Create drop.zip file' - task: ArchiveFiles@2 @@ -2368,7 +2369,7 @@ stages: workingDirectory: $(root_dir) - script: | - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy version`" + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) From aeeba7f166a96b01f1ca03c8326bb2190ebf850d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 19:28:27 +0530 Subject: [PATCH 262/513] test release --- release-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0358a965b..2ac20085f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2042,15 +2042,16 @@ stages: - script: | sudo apt-get install -y zip + chmod +x $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 zip -r $(Build.ArtifactStagingDirectory)/drop.zip . $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version - displayName: 'Create drop.zip file' + displayName: 'Create drop.zip file and verify azcopy' - task: ArchiveFiles@2 displayName: 'Archive all binaries to drop.zip' inputs: rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' - archiveFile: '$(Build.ArtifactStagingDirectory)//drop.zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' continueOnError: true - task: Bash@3 From d8ec621b46bbfcac54185f5983e75bc29b49d883 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 20:47:30 +0530 Subject: [PATCH 263/513] test release --- release-pipelines.yml | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2ac20085f..49d4f8111 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2042,10 +2042,10 @@ stages: - script: | sudo apt-get install -y zip - chmod +x $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.rpm . displayName: 'Create drop.zip file and verify azcopy' + workingDirectory: $(Build.ArtifactStagingDirectory) - task: ArchiveFiles@2 displayName: 'Archive all binaries to drop.zip' @@ -2061,11 +2061,6 @@ stages: script: | curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - script: | - cp $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 . - displayName: 'Copy executable binary to different location' - workingDirectory: $(Build.ArtifactStagingDirectory) - - task: AzureCLI@2 displayName: 'Azure CLI' inputs: @@ -2073,18 +2068,25 @@ stages: scriptType: bash scriptLocation: inlineScript inlineScript: | - # Define variables - artifactDir=$(Build.ArtifactStagingDirectory) - azcopyPath="$artifactDir/azcopy_linux_amd64" + cp $(Build.ArtifactStagingDirectory) - # Ensure azcopy binary is executable - chmod +x $azcopyPath + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + + # # Define variables + # artifactDir=$(Build.ArtifactStagingDirectory) + # azcopyPath="$artifactDir/azcopy_linux_amd64" - # Verify the azcopy binary - if ! $azcopyPath --version; then - echo "Failed to verify azcopy binary." - exit 1 - fi + # # Ensure azcopy binary is executable + # chmod +x $azcopyPath + + # # Verify the azcopy binary + # if ! $azcopyPath --version; then + # echo "Failed to verify azcopy binary." + # exit 1 + # fi # Find the current version raw_version_output=$($(Build.ArtifactStagingDirectory)/azcopy_linux_amd64 --version) From 1d1bff2eeae633333868176dbd5f60f16feb1c11 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 20:56:12 +0530 Subject: [PATCH 264/513] test uts and e2etest --- release-pipelines.yml | 516 +++++++++++++++++++++--------------------- 1 file changed, 260 insertions(+), 256 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 49d4f8111..40e7bf6b0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -49,262 +49,266 @@ trigger: none pr: none stages: - # - stage: RunTests - # jobs: - # - job: UT_On_Ubuntu - # variables: - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' - - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py + - stage: RunTests + jobs: + - job: UT_On_Ubuntu + variables: + isMutexSet: 'false' + # allow maximum build time, in case we have build congestion + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + + steps: + - task: UsePythonVersion@0 + name: 'Set_up_Python' + inputs: + versionSpec: '3.7' + + - task: GoTool@0 + name: 'Set_up_Golang' + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + - task: DownloadSecureFile@1 + name: ciGCSServiceAccountKey + displayName: 'Download GCS Service Account Key' + inputs: + secureFile: 'ci-gcs-dev.json' + + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Install dependencies' + + - script: | + pip install azure-storage-blob==12.12.0 + # set the variable to indicate that the mutex is being acquired + # note: we set it before acquiring the mutex to ensure we release the mutex. + # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # mutex but before we set this variable. + # setting this before will always work since it is valid to break an un-acquired mutex. + echo '##vso[task.setvariable variable=isMutexSet]true' + # acquire the mutex before running live tests to avoid conflicts + python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + name: 'Acquire_the_distributed_mutex' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'cmd' + coverage_name: 'cmd' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common' + coverage_name: 'common' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common/parallel' + coverage_name: 'parallel' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'ste' + coverage_name: 'ste' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'sddl' + coverage_name: 'sddl' + + - script: | + GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + go build -tags "netgo" -o test-validator ./testSuite/ + mkdir test-temp + mkdir coverage + export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + keyctl session test python ./testSuite/scripts/run.py - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) - - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o smoke_coverage.txt + echo 'Formatting coverage to json format' + $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + echo 'Formatting coverage to xml format' + $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + name: 'Run_smoke_tests' + env: + ACCOUNT_NAME: $(ACCOUNT_NAME) + ACCOUNT_KEY: $(ACCOUNT_KEY) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + FILESYSTEM_URL: $(FILESYSTEM_URL) + OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + SHARE_SAS_URL: $(SHARE_SAS_URL) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + condition: succeededOrFailed() + # Smoke Tests Publishing + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + - script: | + pip install azure-storage-blob==12.12.0 + python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + name: 'Release_the_distributed_mutex' + # this runs even if the job was canceled (only if the mutex was acquired by this job) + condition: and(always(), eq(variables['isMutexSet'], 'true')) + - job: E2E_Test + timeoutInMinutes: 360 + # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + build_name: 'azcopy_linux_amd64' + display_name: "Linux" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + Windows: + imageName: 'windows-2019' + build_name: 'azcopy_windows_amd64.exe' + display_name: "Windows" + type: 'windows' + go_path: 'C:\Users\VssAdministrator\go\bin\' + suffix: '.exe' + run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + output_test_logs: 'cat test.txt' + save_exit_code: 'set exit_code=%errorlevel%' + return_exit_code: 'exit %exit_code%' + go_build: 'go build -cover -o $(build_name)' + MacOS: + imageName: 'macos-11' + build_name: 'azcopy_darwin_amd64' + display_name: "MacOS" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + pool: + vmImage: $(imageName) + variables: + - group: AZCOPY_TESTS_VAR + + steps: + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + pwsh: 'true' + displayName: 'Install Powershell Az Module' + - task: GoTool@0 + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Installing dependencies' + - bash: | + echo "##vso[task.setvariable variable=CGO_ENABLED]0" + displayName: 'Set CGO_ENABLED for Windows' + condition: eq(variables.type, 'windows') + - bash: | + npm install -g azurite + mkdir azurite + azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + # Running E2E Tests on AMD64 + - script: | + mkdir coverage + echo 'Building executable' + $(go_build) + echo 'Running tests' + $(run_e2e) + $(save_exit_code) + echo "Generating junit report" + cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + $(output_test_logs) + echo 'Formatting coverage to json format' + $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + echo 'Formatting coverage to xml format' + $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + $(return_exit_code) + env: + AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + displayName: 'E2E Test $(display_name) - AMD64' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish logs' + condition: succeededOrFailed() + inputs: + pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + artifactName: logs + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + testRunTitle: 'Go on $(display_name)' + + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts #dependsOn: RunTests @@ -2069,7 +2073,7 @@ stages: scriptLocation: inlineScript inlineScript: | cp $(Build.ArtifactStagingDirectory) - + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo rpm -qip azcopy-*x86_64.rpm sudo yum groupinstall "Development Tools" -y From 6d54e9204906cad3eea8cd112ebfb8ea20e0d461 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 20:57:19 +0530 Subject: [PATCH 265/513] test uts and e2etest --- release-pipelines.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 40e7bf6b0..3965aa93d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -53,14 +53,12 @@ stages: jobs: - job: UT_On_Ubuntu variables: + - group: AZCOPY_TESTS_VAR isMutexSet: 'false' # allow maximum build time, in case we have build congestion timeoutInMinutes: 360 pool: vmImage: 'ubuntu-20.04' - variables: - - group: AZCOPY_TESTS_VAR - steps: - task: UsePythonVersion@0 name: 'Set_up_Python' From 5845c1485bc9e22d3f8e645e39f28041cf7b6251 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 20:57:57 +0530 Subject: [PATCH 266/513] test uts and e2etest --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 3965aa93d..f8f27020f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -54,7 +54,7 @@ stages: - job: UT_On_Ubuntu variables: - group: AZCOPY_TESTS_VAR - isMutexSet: 'false' + - isMutexSet: 'false' # allow maximum build time, in case we have build congestion timeoutInMinutes: 360 pool: From 108909ddd0523644f067fff5e34a5c8067e5f104 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 17 Jun 2024 21:06:16 +0530 Subject: [PATCH 267/513] test uts and e2etest --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f8f27020f..a57e231a2 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -53,8 +53,8 @@ stages: jobs: - job: UT_On_Ubuntu variables: - - group: AZCOPY_TESTS_VAR - - isMutexSet: 'false' + group: AZCOPY_TESTS_VAR + isMutexSet: 'false' # allow maximum build time, in case we have build congestion timeoutInMinutes: 360 pool: From f7755126ee6b4764160f92ff36efb38322832886 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 10:59:22 +0530 Subject: [PATCH 268/513] test release --- release-pipelines.yml | 536 +++++++++++++++++++++--------------------- 1 file changed, 267 insertions(+), 269 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a57e231a2..2fc441380 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -49,264 +49,264 @@ trigger: none pr: none stages: - - stage: RunTests - jobs: - - job: UT_On_Ubuntu - variables: - group: AZCOPY_TESTS_VAR - isMutexSet: 'false' - # allow maximum build time, in case we have build congestion - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - name: 'Set_up_Python' - inputs: - versionSpec: '3.7' - - - task: GoTool@0 - name: 'Set_up_Golang' - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - - task: DownloadSecureFile@1 - name: ciGCSServiceAccountKey - displayName: 'Download GCS Service Account Key' - inputs: - secureFile: 'ci-gcs-dev.json' - - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Install dependencies' - - - script: | - pip install azure-storage-blob==12.12.0 - # set the variable to indicate that the mutex is being acquired - # note: we set it before acquiring the mutex to ensure we release the mutex. - # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # mutex but before we set this variable. - # setting this before will always work since it is valid to break an un-acquired mutex. - echo '##vso[task.setvariable variable=isMutexSet]true' - # acquire the mutex before running live tests to avoid conflicts - python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - name: 'Acquire_the_distributed_mutex' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'cmd' - coverage_name: 'cmd' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common' - coverage_name: 'common' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common/parallel' - coverage_name: 'parallel' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'ste' - coverage_name: 'ste' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'sddl' - coverage_name: 'sddl' - - - script: | - GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - go build -tags "netgo" -o test-validator ./testSuite/ - mkdir test-temp - mkdir coverage - export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - keyctl session test python ./testSuite/scripts/run.py + # - stage: RunTests + # jobs: + # - job: UT_On_Ubuntu + # variables: + # group: AZCOPY_TESTS_VAR + # isMutexSet: 'false' + # # allow maximum build time, in case we have build congestion + # timeoutInMinutes: 360 + # pool: + # vmImage: 'ubuntu-20.04' + # steps: + # - task: UsePythonVersion@0 + # name: 'Set_up_Python' + # inputs: + # versionSpec: '3.7' + + # - task: GoTool@0 + # name: 'Set_up_Golang' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + # - task: DownloadSecureFile@1 + # name: ciGCSServiceAccountKey + # displayName: 'Download GCS Service Account Key' + # inputs: + # secureFile: 'ci-gcs-dev.json' + + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Install dependencies' + + # - script: | + # pip install azure-storage-blob==12.12.0 + # # set the variable to indicate that the mutex is being acquired + # # note: we set it before acquiring the mutex to ensure we release the mutex. + # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # # mutex but before we set this variable. + # # setting this before will always work since it is valid to break an un-acquired mutex. + # echo '##vso[task.setvariable variable=isMutexSet]true' + # # acquire the mutex before running live tests to avoid conflicts + # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + # name: 'Acquire_the_distributed_mutex' + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'cmd' + # coverage_name: 'cmd' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common' + # coverage_name: 'common' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common/parallel' + # coverage_name: 'parallel' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'ste' + # coverage_name: 'ste' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'sddl' + # coverage_name: 'sddl' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + # go build -tags "netgo" -o test-validator ./testSuite/ + # mkdir test-temp + # mkdir coverage + # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + # export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + # keyctl session test python ./testSuite/scripts/run.py - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o smoke_coverage.txt - echo 'Formatting coverage to json format' - $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - echo 'Formatting coverage to xml format' - $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - name: 'Run_smoke_tests' - env: - ACCOUNT_NAME: $(ACCOUNT_NAME) - ACCOUNT_KEY: $(ACCOUNT_KEY) - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - FILESYSTEM_URL: $(FILESYSTEM_URL) - OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - SHARE_SAS_URL: $(SHARE_SAS_URL) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - condition: succeededOrFailed() - # Smoke Tests Publishing - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - - script: | - pip install azure-storage-blob==12.12.0 - python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - name: 'Release_the_distributed_mutex' - # this runs even if the job was canceled (only if the mutex was acquired by this job) - condition: and(always(), eq(variables['isMutexSet'], 'true')) - - job: E2E_Test - timeoutInMinutes: 360 - # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - strategy: - matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - build_name: 'azcopy_linux_amd64' - display_name: "Linux" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - Windows: - imageName: 'windows-2019' - build_name: 'azcopy_windows_amd64.exe' - display_name: "Windows" - type: 'windows' - go_path: 'C:\Users\VssAdministrator\go\bin\' - suffix: '.exe' - run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - output_test_logs: 'cat test.txt' - save_exit_code: 'set exit_code=%errorlevel%' - return_exit_code: 'exit %exit_code%' - go_build: 'go build -cover -o $(build_name)' - MacOS: - imageName: 'macos-11' - build_name: 'azcopy_darwin_amd64' - display_name: "MacOS" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - pool: - vmImage: $(imageName) - variables: - - group: AZCOPY_TESTS_VAR - - steps: - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - pwsh: 'true' - displayName: 'Install Powershell Az Module' - - task: GoTool@0 - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Installing dependencies' - - bash: | - echo "##vso[task.setvariable variable=CGO_ENABLED]0" - displayName: 'Set CGO_ENABLED for Windows' - condition: eq(variables.type, 'windows') - - bash: | - npm install -g azurite - mkdir azurite - azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' - # Running E2E Tests on AMD64 - - script: | - mkdir coverage - echo 'Building executable' - $(go_build) - echo 'Running tests' - $(run_e2e) - $(save_exit_code) - echo "Generating junit report" - cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - $(output_test_logs) - echo 'Formatting coverage to json format' - $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - echo 'Formatting coverage to xml format' - $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - $(return_exit_code) - env: - AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - displayName: 'E2E Test $(display_name) - AMD64' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish logs' - condition: succeededOrFailed() - inputs: - pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - artifactName: logs - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testRunner: JUnit - testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - testRunTitle: 'Go on $(display_name)' - - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o smoke_coverage.txt + # echo 'Formatting coverage to json format' + # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + # echo 'Formatting coverage to xml format' + # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + # name: 'Run_smoke_tests' + # env: + # ACCOUNT_NAME: $(ACCOUNT_NAME) + # ACCOUNT_KEY: $(ACCOUNT_KEY) + # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + # FILESYSTEM_URL: $(FILESYSTEM_URL) + # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + # SHARE_SAS_URL: $(SHARE_SAS_URL) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # condition: succeededOrFailed() + # # Smoke Tests Publishing + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + # - script: | + # pip install azure-storage-blob==12.12.0 + # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + # name: 'Release_the_distributed_mutex' + # # this runs even if the job was canceled (only if the mutex was acquired by this job) + # condition: and(always(), eq(variables['isMutexSet'], 'true')) + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) + # variables: + # - group: AZCOPY_TESTS_VAR + + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' + + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs + + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' + + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts #dependsOn: RunTests @@ -1998,16 +1998,15 @@ stages: condition: succeeded('TestArtifacts') jobs: - job: ReleaseToContainer - timeoutInMinutes: 120 strategy: matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + vmImage: $(vmImage) + variables: - name: Test_Release_Account_Name value: 'azcopyprivatedrops' @@ -2070,12 +2069,11 @@ stages: scriptType: bash scriptLocation: inlineScript inlineScript: | - cp $(Build.ArtifactStagingDirectory) + cd $(Build.ArtifactStagingDirectory) - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y # # Define variables # artifactDir=$(Build.ArtifactStagingDirectory) From 3dea4624340aafc2ebc0806161f9a6fb3caf17bc Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 11:36:29 +0530 Subject: [PATCH 269/513] test release --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2fc441380..69b44b41c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2044,7 +2044,7 @@ stages: - script: | sudo apt-get install -y zip zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.rpm . + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . displayName: 'Create drop.zip file and verify azcopy' workingDirectory: $(Build.ArtifactStagingDirectory) From 2f96a01dd4c4481d70d2869200d7933ba7795bf3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 12:04:19 +0530 Subject: [PATCH 270/513] test release --- release-pipelines.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 69b44b41c..7d9ae3bf5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2075,18 +2075,7 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y - # # Define variables - # artifactDir=$(Build.ArtifactStagingDirectory) - # azcopyPath="$artifactDir/azcopy_linux_amd64" - - # # Ensure azcopy binary is executable - # chmod +x $azcopyPath - - # # Verify the azcopy binary - # if ! $azcopyPath --version; then - # echo "Failed to verify azcopy binary." - # exit 1 - # fi + ls -ltR $(Build.ArtifactStagingDirectory)/ # Find the current version raw_version_output=$($(Build.ArtifactStagingDirectory)/azcopy_linux_amd64 --version) From d2e741d4b248f1e61f58553edc9ab6595224331f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 12:39:54 +0530 Subject: [PATCH 271/513] test release --- release-pipelines.yml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7d9ae3bf5..18616e22a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2045,6 +2045,10 @@ stages: sudo apt-get install -y zip zip -r $(Build.ArtifactStagingDirectory)/drop.zip . cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + ls -ltR $(Build.ArtifactStagingDirectory)/ displayName: 'Create drop.zip file and verify azcopy' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -2069,16 +2073,8 @@ stages: scriptType: bash scriptLocation: inlineScript inlineScript: | - cd $(Build.ArtifactStagingDirectory) - - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - - ls -ltR $(Build.ArtifactStagingDirectory)/ - # Find the current version - raw_version_output=$($(Build.ArtifactStagingDirectory)/azcopy_linux_amd64 --version) + raw_version_output=$(azcopy_linux_amd64 --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') today=$(date +"%Y%m%d") @@ -2093,10 +2089,10 @@ stages: executable_name=azcopy_linux_amd64 # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $(Build.ArtifactStagingDirectory)/$executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 @@ -2327,6 +2323,7 @@ stages: pool: vmImage: 'ubuntu-20.04' variables: + - group: AZCOPY_TESTS_VAR - name: root_dir value: '$(System.DefaultWorkingDirectory)' From daedbc5420610b24d3afc2e1ae7ab702a4a06c0a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 13:44:56 +0530 Subject: [PATCH 272/513] test release --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 18616e22a..af2151fa4 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2079,7 +2079,7 @@ stages: today=$(date +"%Y%m%d") - if [ "$test_mode" = "true" ]; then + if [ "${{ parameters.test_mode }}" = "true" ]; then container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" else container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" From 6f84c15d7709e929307d193ddb106de2827e7f3a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 15:34:10 +0530 Subject: [PATCH 273/513] test signing --- release-pipelines.yml | 913 ++++++++++++----------------- setup/esrp_sign_linux.yml | 25 + setup/esrp_sign_notarize_macos.yml | 59 ++ setup/esrp_sign_windows.yml | 50 ++ setup/releaseVersionUpdate.py | 49 ++ 5 files changed, 548 insertions(+), 548 deletions(-) create mode 100644 setup/esrp_sign_linux.yml create mode 100644 setup/esrp_sign_notarize_macos.yml create mode 100644 setup/esrp_sign_windows.yml create mode 100644 setup/releaseVersionUpdate.py diff --git a/release-pipelines.yml b/release-pipelines.yml index af2151fa4..0d503fa66 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -390,14 +390,10 @@ stages: - script: | sudo ls -lRt $(linux_amd64)/ sudo ls -lRt $(linux_se_amd64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - task: PublishBuildArtifacts@1 inputs: @@ -415,7 +411,7 @@ stages: cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ cp NOTICE.txt $(linux_amd64)/ cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files' + displayName: 'Copy required files for packaging' - script: | sudo ls -lRt pkgDir/usr/bin/ @@ -584,14 +580,10 @@ stages: - script: | sudo ls -lRt $(linux_arm64)/ sudo ls -lRt $(linux_se_arm64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - task: PublishBuildArtifacts@1 inputs: @@ -610,10 +602,6 @@ stages: cp NOTICE.txt $(linux_se_arm64)/ displayName: 'Copy required files' - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - script: | export GOARCH=arm64 export GOOS=linux @@ -945,43 +933,19 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' - Pattern: '*.rpm, *.deb, *tar.gz' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' - Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + - template: esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' # Validate signed images have md5sum changed - script: | @@ -1073,50 +1037,11 @@ stages: sudo apt-get install -y zip displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + - template: esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) Pattern: '*.exe' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 # Validate signed images have md5sum changed - script: | @@ -1235,57 +1160,11 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + - template: esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) Pattern: '*.zip' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Notarize Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppNotarize", - "Parameters":{ - "BundleId":"com.microsoft.AzCopy" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 # Validate signed images have md5sum changed - script: | @@ -1313,7 +1192,7 @@ stages: mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ rm -rf $(Build.ArtifactStagingDirectory)/unzipped sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac build to staging directory' + displayName: 'Copy signed mac binary to staging directory' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 @@ -1365,71 +1244,19 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - # Send linux images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' + - template: esrp_sign_linux.yml + parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: 'azcopy_linux*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: esrp_sign_windows.yml + parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: '*.exe' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 - + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* @@ -1525,397 +1352,397 @@ stages: azcopy_linux_amd64 --help displayName: 'Check Help' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # # TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + # TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*arm64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*arm64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # azcopy_linux_arm64 --version - # displayName: 'Check Version' + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' - # - script: | - # azcopy_linux_arm64 --help - # displayName: 'Check Help' + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed - - # - script: | - # azcopy_linux_arm64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_arm64 --help - # displayName: 'Check Help' + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed + + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed - - # - script: | - # azcopy_linux_amd64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_amd64 --help - # displayName: 'Check Help' - - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' - # steps: - # - checkout: none + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + steps: + - checkout: none - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # azcopy_linux_amd64 --version - # displayName: 'Check Version' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm - # - script: | - # azcopy_linux_amd64 --help - # displayName: 'Check Help' + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) - # # # Install zip and unzip utilities - # # brew install zip unzip + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # # Install zip and unzip utilities + # brew install zip unzip - # # Create extraction directory - # mkdir -p $EXTRACT_DIR + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # Create extraction directory + mkdir -p $EXTRACT_DIR - # # Change to the directory containing azcopy - # cd $EXTRACT_DIR/azcopy_darwin_amd64 + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # # Run the azcopy version command - # ./azcopy_darwin_amd64 --version - # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - # - job: Set_7 + - job: Set_7 # timeoutInMinutes: 120 # strategy: # matrix: @@ -2048,7 +1875,6 @@ stages: sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y - ls -ltR $(Build.ArtifactStagingDirectory)/ displayName: 'Create drop.zip file and verify azcopy' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -2249,11 +2075,7 @@ stages: displayName: 'List Artifacts' - script: | - mv azCopy-linux-signed/azcopy_linux_se_amd64 . - mv azCopy-linux-signed/azcopy_linux_se_arm64 . pmc package upload azCopy-linux-signed - mv azcopy_linux_se_amd64 azCopy-linux-signed/ - mv azcopy_linux_se_arm64 azCopy-linux-signed/ displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -2264,8 +2086,6 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - mv azcopy_linux_se_amd64 ../ - mv azcopy_linux_se_arm64 ../ for file in "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed"/*; do if [ -f "$file" ]; then @@ -2296,9 +2116,6 @@ stages: echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) - - mv ../azcopy_linux_se_amd64 . - mv ../azcopy_linux_se_arm64 . displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ @@ -2358,7 +2175,7 @@ stages: workingDirectory: $(root_dir) - script: | - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + python3 setup/releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) diff --git a/setup/esrp_sign_linux.yml b/setup/esrp_sign_linux.yml new file mode 100644 index 000000000..949238cc0 --- /dev/null +++ b/setup/esrp_sign_linux.yml @@ -0,0 +1,25 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_KEY_CODE: '' + Pattern: '' + DisplayName: '' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ${{ parameters.DisplayName }} + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "${{ parameters.ESRP_AZCOPY_KEY_CODE }}", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] diff --git a/setup/esrp_sign_notarize_macos.yml b/setup/esrp_sign_notarize_macos.yml new file mode 100644 index 000000000..543844076 --- /dev/null +++ b/setup/esrp_sign_notarize_macos.yml @@ -0,0 +1,59 @@ +# esrp-sign-notarize-macos.yml +parameters: + FolderPath: '' + ESRP_AZCOPY_MAC_KEY_CODE: '' + Pattern: '*.zip' + + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Notarize Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 diff --git a/setup/esrp_sign_windows.yml b/setup/esrp_sign_windows.yml new file mode 100644 index 000000000..1aeb8ee68 --- /dev/null +++ b/setup/esrp_sign_windows.yml @@ -0,0 +1,50 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_WIN_KEY_CODE: '' + Pattern: '*.exe' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "${{ parameters.ESRP_AZCOPY_WIN_KEY_CODE }}", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py new file mode 100644 index 000000000..f00c2f57e --- /dev/null +++ b/setup/releaseVersionUpdate.py @@ -0,0 +1,49 @@ +import json +import requests +import sys +import os +from xml.dom import minidom +from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient + + +sasUrl = sys.argv[1] +releaseVersion = sys.argv[2].split(' ')[2] +print('Release Version: ' + releaseVersion) +if(len(releaseVersion)==0): + print('Incorrect Release Version') + sys.exit(1) + +containerUrl = sasUrl.split('?')[0] +sasToken = sasUrl.split('?')[1] + +# delete latest version file in the container +deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +resp = requests.delete(deleteUrl) +sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') + +# Create a file and write data to it +file_name = 'latest_version.txt' + +# Open the file in write mode +with open(file_name, 'w') as file: + file.write(releaseVersion) +print(f'Data written to {file_name}') + +# Create the BlobServiceClient using the SAS URL +blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) + +# Get the container name from the SAS URL +container_name = sasUrl.split('?')[0].split('/')[-1] + +# Get a client to interact with the container +container_client = blob_service_client.get_container_client(container_name) + +# Upload the file to the container +with open(file_name, 'rb') as data: + container_client.upload_blob(name=file_name, data=data, overwrite=True) + +print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + +# Clean up the local file +os.remove(file_name) +print(f"Local file '{file_name}' deleted.") From 2c78443cddf93861b36bec2e4a4e45ba1cb44ade Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 15:34:10 +0530 Subject: [PATCH 274/513] test signing --- release-pipelines.yml | 909 ++++++++++++----------------- setup/esrp_sign_linux.yml | 25 + setup/esrp_sign_notarize_macos.yml | 59 ++ setup/esrp_sign_windows.yml | 50 ++ setup/releaseVersionUpdate.py | 49 ++ 5 files changed, 546 insertions(+), 546 deletions(-) create mode 100644 setup/esrp_sign_linux.yml create mode 100644 setup/esrp_sign_notarize_macos.yml create mode 100644 setup/esrp_sign_windows.yml create mode 100644 setup/releaseVersionUpdate.py diff --git a/release-pipelines.yml b/release-pipelines.yml index af2151fa4..f074c4deb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -390,14 +390,10 @@ stages: - script: | sudo ls -lRt $(linux_amd64)/ sudo ls -lRt $(linux_se_amd64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - task: PublishBuildArtifacts@1 inputs: @@ -415,7 +411,7 @@ stages: cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ cp NOTICE.txt $(linux_amd64)/ cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files' + displayName: 'Copy required files for packaging' - script: | sudo ls -lRt pkgDir/usr/bin/ @@ -584,14 +580,10 @@ stages: - script: | sudo ls -lRt $(linux_arm64)/ sudo ls -lRt $(linux_se_arm64)/ - displayName: 'List the binaries' - workingDirectory: $(root_dir) - - - script: | cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' workingDirectory: $(root_dir) - displayName: 'Add binaries to staging directory' - task: PublishBuildArtifacts@1 inputs: @@ -610,10 +602,6 @@ stages: cp NOTICE.txt $(linux_se_arm64)/ displayName: 'Copy required files' - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - script: | export GOARCH=arm64 export GOOS=linux @@ -945,43 +933,19 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-linux-temp' + - template: esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: '*.rpm, *.deb, *tar.gz' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy mariner' - inputs: - ConnectedServiceName: 'PMC ESRP azCopy Signing' - FolderPath: '$(Build.ArtifactStagingDirectory)/mariner' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) Pattern: '*.rpm, *.deb' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_MARINER_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] + DisplayName: 'ESRP CodeSigning azCopy mariner' # Validate signed images have md5sum changed - script: | @@ -1073,50 +1037,11 @@ stages: sudo apt-get install -y zip displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-windows-temp' + - template: esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) Pattern: '*.exe' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 # Validate signed images have md5sum changed - script: | @@ -1235,57 +1160,11 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' + - template: esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) Pattern: '*.zip' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"$(ESRP_AZCOPY_MAC_KEY_CODE)", - "OperationCode":"MacAppDeveloperSign", - "Parameters":{ - "Hardening":"--options=runtime" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Notarize Mac executable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-mac-temp' - Pattern: 'azcopy_darwin*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode":"CP-401337-Apple", - "OperationCode":"MacAppNotarize", - "Parameters":{ - "BundleId":"com.microsoft.AzCopy" - }, - "ToolName":"sign", - "ToolVersion":"1.0" - } - ] - SessionTimeout: 30 # Validate signed images have md5sum changed - script: | @@ -1313,7 +1192,7 @@ stages: mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ rm -rf $(Build.ArtifactStagingDirectory)/unzipped sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac build to staging directory' + displayName: 'Copy signed mac binary to staging directory' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 @@ -1365,71 +1244,19 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - # Send linux images for signing - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning azCopy linux' - inputs: - ConnectedServiceName: 'PMC ESRP AzCopy Signing' + - template: esrp_sign_linux.yml + parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: 'azcopy_linux*' - signConfigType: inlineSignParams - VerboseLogin: true - inlineOperation: | - [ - { - "KeyCode" : "$(ESRP_AZCOPY_KEY_CODE)", - "OperationCode" : "LinuxSign", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 - displayName: 'Sign Windows exectuable' - inputs: - ConnectedServiceName: 'ESRP KeyVault identity' - AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'azcopy-esrp-kv' - AuthCertName: 'azcopy-auth' - AuthSignCertName: 'azcopy-sign-test' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: esrp_sign_windows.yml + parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: '*.exe' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "$(ESRP_AZCOPY_WIN_KEY_CODE)", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd sha256" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "signtool.exe", - "toolVersion": "6.2.9304.0" - } - ] - SessionTimeout: 30 - + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* @@ -1525,397 +1352,397 @@ stages: azcopy_linux_amd64 --help displayName: 'Check Help' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # # TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + # TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*arm64.deb + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*arm64.deb - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # azcopy_linux_arm64 --version - # displayName: 'Check Version' + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' - # - script: | - # azcopy_linux_arm64 --help - # displayName: 'Check Help' + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed - - # - script: | - # azcopy_linux_arm64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_arm64 --help - # displayName: 'Check Help' + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed + + - script: | + azcopy_linux_arm64 --version + displayName: 'Check Version' + + - script: | + azcopy_linux_arm64 --help + displayName: 'Check Help' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed - - # - script: | - # azcopy_linux_amd64 --version - # displayName: 'Check Version' - - # - script: | - # azcopy_linux_amd64 --help - # displayName: 'Check Help' - - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' - # steps: - # - checkout: none + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) - # #TODO: replace org name from dphulkar-msft to Azure - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(root_dir)/azure-storage-azcopy + steps: + - checkout: none - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "GoTool Custom Setup" + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + #TODO: replace org name from dphulkar-msft to Azure + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(root_dir) - # itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy - # - script: | - # ls -l - # result=$(ls -1 | wc -l) - # if [ $result -ne 1 ]; then - # exit 1 - # fi - # displayName: 'List Downloaded Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # azcopy_linux_amd64 --version - # displayName: 'Check Version' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(root_dir) + itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm - # - script: | - # azcopy_linux_amd64 --help - # displayName: 'Check Help' + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(root_dir)/azCopy-linux-signed/mariner - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - script: | + azcopy_linux_amd64 --version + displayName: 'Check Version' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + - script: | + azcopy_linux_amd64 --help + displayName: 'Check Help' - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) - # # # Install zip and unzip utilities - # # brew install zip unzip + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # # Install zip and unzip utilities + # brew install zip unzip - # # Create extraction directory - # mkdir -p $EXTRACT_DIR + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # Create extraction directory + mkdir -p $EXTRACT_DIR - # # Change to the directory containing azcopy - # cd $EXTRACT_DIR/azcopy_darwin_amd64 + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # # Run the azcopy version command - # ./azcopy_darwin_amd64 --version - # displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' - # - job: Set_7 + - job: Set_7 # timeoutInMinutes: 120 # strategy: # matrix: @@ -2048,7 +1875,6 @@ stages: sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y - ls -ltR $(Build.ArtifactStagingDirectory)/ displayName: 'Create drop.zip file and verify azcopy' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -2249,11 +2075,7 @@ stages: displayName: 'List Artifacts' - script: | - mv azCopy-linux-signed/azcopy_linux_se_amd64 . - mv azCopy-linux-signed/azcopy_linux_se_arm64 . pmc package upload azCopy-linux-signed - mv azcopy_linux_se_amd64 azCopy-linux-signed/ - mv azcopy_linux_se_arm64 azCopy-linux-signed/ displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -2264,8 +2086,6 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - mv azcopy_linux_se_amd64 ../ - mv azcopy_linux_se_arm64 ../ for file in "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed"/*; do if [ -f "$file" ]; then @@ -2296,9 +2116,6 @@ stages: echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) - - mv ../azcopy_linux_se_amd64 . - mv ../azcopy_linux_se_arm64 . displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ @@ -2358,7 +2175,7 @@ stages: workingDirectory: $(root_dir) - script: | - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + python3 setup/releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) diff --git a/setup/esrp_sign_linux.yml b/setup/esrp_sign_linux.yml new file mode 100644 index 000000000..949238cc0 --- /dev/null +++ b/setup/esrp_sign_linux.yml @@ -0,0 +1,25 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_KEY_CODE: '' + Pattern: '' + DisplayName: '' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ${{ parameters.DisplayName }} + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "${{ parameters.ESRP_AZCOPY_KEY_CODE }}", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] diff --git a/setup/esrp_sign_notarize_macos.yml b/setup/esrp_sign_notarize_macos.yml new file mode 100644 index 000000000..543844076 --- /dev/null +++ b/setup/esrp_sign_notarize_macos.yml @@ -0,0 +1,59 @@ +# esrp-sign-notarize-macos.yml +parameters: + FolderPath: '' + ESRP_AZCOPY_MAC_KEY_CODE: '' + Pattern: '*.zip' + + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Notarize Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 diff --git a/setup/esrp_sign_windows.yml b/setup/esrp_sign_windows.yml new file mode 100644 index 000000000..1aeb8ee68 --- /dev/null +++ b/setup/esrp_sign_windows.yml @@ -0,0 +1,50 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_WIN_KEY_CODE: '' + Pattern: '*.exe' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "${{ parameters.ESRP_AZCOPY_WIN_KEY_CODE }}", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py new file mode 100644 index 000000000..f00c2f57e --- /dev/null +++ b/setup/releaseVersionUpdate.py @@ -0,0 +1,49 @@ +import json +import requests +import sys +import os +from xml.dom import minidom +from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient + + +sasUrl = sys.argv[1] +releaseVersion = sys.argv[2].split(' ')[2] +print('Release Version: ' + releaseVersion) +if(len(releaseVersion)==0): + print('Incorrect Release Version') + sys.exit(1) + +containerUrl = sasUrl.split('?')[0] +sasToken = sasUrl.split('?')[1] + +# delete latest version file in the container +deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +resp = requests.delete(deleteUrl) +sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') + +# Create a file and write data to it +file_name = 'latest_version.txt' + +# Open the file in write mode +with open(file_name, 'w') as file: + file.write(releaseVersion) +print(f'Data written to {file_name}') + +# Create the BlobServiceClient using the SAS URL +blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) + +# Get the container name from the SAS URL +container_name = sasUrl.split('?')[0].split('/')[-1] + +# Get a client to interact with the container +container_client = blob_service_client.get_container_client(container_name) + +# Upload the file to the container +with open(file_name, 'rb') as data: + container_client.upload_blob(name=file_name, data=data, overwrite=True) + +print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + +# Clean up the local file +os.remove(file_name) +print(f"Local file '{file_name}' deleted.") From 8d0a2378d27a94f2c3916cfa7b397667f1eb85cb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 15:40:35 +0530 Subject: [PATCH 275/513] test sign --- release-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f074c4deb..4ae9271fc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -933,14 +933,14 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - template: esrp_sign_linux.yml + - template: setup/esrp_sign_linux.yml parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: '*.rpm, *.deb, *tar.gz' DisplayName: 'ESRP CodeSigning azCopy linux' - - template: esrp_sign_linux.yml + - template: setup/esrp_sign_linux.yml parameters: FolderPath: $(Build.ArtifactStagingDirectory)/mariner ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) @@ -1037,7 +1037,7 @@ stages: sudo apt-get install -y zip displayName: "Update dependencies" - - template: esrp_sign_windows.yml + - template: setup/esrp_sign_windows.yml parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) @@ -1160,7 +1160,7 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - template: esrp_sign_notarize_macos.yml + - template: setup/esrp_sign_notarize_macos.yml parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) @@ -1244,14 +1244,14 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - template: esrp_sign_linux.yml + - template: setup/esrp_sign_linux.yml parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) Pattern: 'azcopy_linux*' DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - template: esrp_sign_windows.yml + - template: setup/esrp_sign_windows.yml parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) From a8737e4653195ebefac8e79246639a4cc0239e40 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 18:10:37 +0530 Subject: [PATCH 276/513] test sign --- release-pipelines.yml | 216 ++++++++++++++++++++---------------------- 1 file changed, 101 insertions(+), 115 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4ae9271fc..06e1dc06a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -14,10 +14,10 @@ parameters: type: boolean default: false - - name: unit_test - displayName: 'Execute Unit Tests' - type: boolean - default: false + # - name: unit_test + # displayName: 'Execute Unit Tests' + # type: boolean + # default: false - name: post_release displayName: 'Post Release on Github' @@ -1204,7 +1204,7 @@ stages: timeoutInMinutes: 120 strategy: matrix: - Linux-Signing: + Binaries-Signing: vmImage: 'Ubuntu-20.04' AgentName: "blobfuse-ubuntu20" pool: @@ -1254,7 +1254,7 @@ stages: - template: setup/esrp_sign_windows.yml parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) Pattern: '*.exe' # Validate signed images have md5sum changed @@ -1301,18 +1301,12 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - checkout: none - - #TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - + #TODO: remove this after merging to main - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' # Custom script to install Go-lang - task: ShellScript@2 @@ -1372,13 +1366,12 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - checkout: none - - # TODO: replace org name from dphulkar-msft to Azure + #TODO: remove this after merging to main - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` @@ -1392,7 +1385,6 @@ stages: args: "$(root_dir)/" displayName: "GoTool Custom Setup" - # get glibc version with which build is done - script: | sudo apt update sudo apt --fix-broken install @@ -1450,18 +1442,17 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - checkout: none - + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' - - #TODO: replace org name from dphulkar-msft to Azure - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` @@ -1533,18 +1524,17 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - checkout: none - - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' - #TODO: replace org name from dphulkar-msft to Azure + #TODO: remove this after merging to main - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` @@ -1617,18 +1607,17 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' steps: - - checkout: none - - script: | sudo tdnf update -y sudo tdnf install git -y displayName: 'Install Git' - #TODO: replace org name from dphulkar-msft to Azure + #TODO: remove this after merging to main - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` @@ -1722,9 +1711,6 @@ stages: eval "$(/opt/homebrew/bin/brew shellenv)" fi - # # Install zip and unzip utilities - # brew install zip unzip - # Define variables ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" @@ -1740,83 +1726,83 @@ stages: # Run the azcopy version command ./azcopy_darwin_amd64 --version - displayName: 'Install Zip, Extract Files, and Run Version Command on macOS' + displayName: 'Install Zip, Extract Files, and Run Version Command' - job: Set_7 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # # Find the zip file matching the pattern - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } - # $zipFilePath = $zipFile.FullName + $zipFilePath = $zipFile.FullName - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir + # Change to the directory containing azcopy + Set-Location -Path $extractDir - # # Verify that azcopy.exe exists - # if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { - # Write-Error "azcopy.exe not found in extraction directory: $extractDir" - # exit 1 - # } + # Verify that azcopy.exe exists + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { + Write-Error "azcopy.exe not found in extraction directory: $extractDir" + exit 1 + } - # # Run the azcopy version command - # .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version - # displayName: 'Extract Files and Run Version Command on Windows' + # Run the azcopy version command + .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version + displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1839,7 +1825,7 @@ stages: value: 'azcopyprivatedrops' - name: Release_Account_Name value: 'azcopyvnextrelease' - - name: test_mode + - name: Test_Mode value: ${{ parameters.test_mode }} steps: @@ -1875,7 +1861,7 @@ stages: sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y - displayName: 'Create drop.zip file and verify azcopy' + displayName: 'Create drop.zip file and install azcopy deb package' workingDirectory: $(Build.ArtifactStagingDirectory) - task: ArchiveFiles@2 @@ -1905,7 +1891,7 @@ stages: today=$(date +"%Y%m%d") - if [ "${{ parameters.test_mode }}" = "true" ]; then + if [ "$(Test_Mode)" = "true" ]; then container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" else container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" From e05c90aa437732286634e55e7fb7a94796b42ff8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:04:48 +0530 Subject: [PATCH 277/513] test sign --- release-pipelines.yml | 251 +++++++----------------------------------- 1 file changed, 40 insertions(+), 211 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 06e1dc06a..71f1aacc5 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1254,7 +1254,7 @@ stages: - template: setup/esrp_sign_windows.yml parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) Pattern: '*.exe' # Validate signed images have md5sum changed @@ -1292,51 +1292,25 @@ stages: container: 'test-cnt-ubn-22' pool: vmImage: $(vmImage) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - #TODO: remove this after merging to main - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*x86_64.deb + - template: test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - script: | sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Install deb Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | azcopy_linux_amd64 --version @@ -1359,59 +1333,24 @@ stages: demands: - ImageOverride -equals $(AgentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version + sudo apt update + sudo apt --fix-broken install + ldd --version displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*arm64.deb - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + - template: test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - script: | sudo dpkg --info azcopy-*arm64.deb sudo dpkg -i azcopy-*arm64.deb displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | azcopy_linux_arm64 --version @@ -1435,57 +1374,17 @@ stages: demands: - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*arm64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + - template: test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo @@ -1493,7 +1392,7 @@ stages: sudo yum groupinstall "Development Tools" -y sudo rpm -i azcopy-*.arm64.rpm displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | azcopy_linux_arm64 --version @@ -1517,65 +1416,30 @@ stages: demands: - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - + - template: test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # get glibc version with which build is done - script: | ldd --version displayName: "GLIBC Version" - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/azcopy-*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed - - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo rpm -qip azcopy-*x86_64.rpm sudo yum groupinstall "Development Tools" -y sudo rpm -i azcopy-*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | azcopy_linux_amd64 --version @@ -1600,63 +1464,28 @@ stages: demands: - ImageOverride -equals $(AgentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - script: | sudo tdnf update -y sudo tdnf install git -y displayName: 'Install Git' - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" + - template: test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed/mariner' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' # get glibc version with which build is done - script: | ldd --version - displayName: "GLIBC Version" - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(root_dir) - itemPattern: azCopy-linux-signed/mariner/azcopy*x86_64.rpm - - - script: | - ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi - displayName: 'List Downloaded Package' - workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + displayName: "GLIBC Version" - script: | sudo rpm -qip azcopy*x86_64.rpm sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(root_dir)/azCopy-linux-signed/mariner + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | azcopy_linux_amd64 --version From 0d602a9deb5bb1c22e5a2019cbe5b8fdf7c1824d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:09:02 +0530 Subject: [PATCH 278/513] test --- setup/test_artifacts.yml | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 setup/test_artifacts.yml diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml new file mode 100644 index 000000000..d5b598f96 --- /dev/null +++ b/setup/test_artifacts.yml @@ -0,0 +1,47 @@ +parameters: + - name: artifact_name + type: string + - name: download_path + type: string + - name: item_pattern + type: string + +variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + +steps: + #TODO: Remove this while merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # displayName: 'List files after downloading go_installer.sh' + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy + + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: ${{ parameters.artifact_name }} + downloadPath: ${{ parameters.download_path }} + itemPattern: ${{ parameters.item_pattern }} + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} From 155ad1c2539bcce17ec15446427058439054d31a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:12:10 +0530 Subject: [PATCH 279/513] test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 71f1aacc5..62b56e2e2 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1299,7 +1299,7 @@ stages: ldd --version displayName: "GLIBC Version" - - template: test_artifacts.yml + - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' From c4cdc8e91ed1ee43bb2573e1539c33c967ee19ca Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:19:22 +0530 Subject: [PATCH 280/513] test --- setup/test_artifacts.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index d5b598f96..93efa498b 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -22,12 +22,12 @@ steps: - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy - task: ShellScript@2 inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" + scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" + args: "$(System.DefaultWorkingDirectory)/" displayName: "GoTool Custom Setup" - task: DownloadBuildArtifacts@0 From d35950386dde919279dac2625b4e90ef11945889 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:20:29 +0530 Subject: [PATCH 281/513] test --- setup/test_artifacts.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 93efa498b..528e0fc45 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -6,12 +6,6 @@ parameters: - name: item_pattern type: string -variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: #TODO: Remove this while merging to main # - script: | From 9fc2054aecddec9514220251e7bfeac15fb63612 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:22:00 +0530 Subject: [PATCH 282/513] test --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 62b56e2e2..9e4b871b7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1340,7 +1340,7 @@ stages: ldd --version displayName: "GLIBC Version" - - template: test_artifacts.yml + - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' @@ -1380,7 +1380,7 @@ stages: sudo yum install git -y displayName: 'Install Git' - - template: test_artifacts.yml + - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' @@ -1422,7 +1422,7 @@ stages: sudo yum install git -y displayName: 'Install Git' - - template: test_artifacts.yml + - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' @@ -1470,7 +1470,7 @@ stages: sudo tdnf install git -y displayName: 'Install Git' - - template: test_artifacts.yml + - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed/mariner' download_path: '$(System.DefaultWorkingDirectory)' From c33dadf61e7cb4800d44930a73a87187f25ebbe6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 20:28:00 +0530 Subject: [PATCH 283/513] test runUT --- release-pipelines.yml | 515 +++++++++++++++++++++--------------------- 1 file changed, 258 insertions(+), 257 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9e4b871b7..cf8178083 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -2,6 +2,7 @@ variables: AZCOPY_GOLANG_VERSION: '1.21.8' AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' AZCOPY_VERSION: '0.0.0' + MUTEX_URL: https://ciazcopyv10regular.blob.core.windows.net/distributedmutex/lock parameters: - name: tag @@ -49,264 +50,264 @@ trigger: none pr: none stages: - # - stage: RunTests - # jobs: - # - job: UT_On_Ubuntu - # variables: - # group: AZCOPY_TESTS_VAR - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py + - stage: RunTests + jobs: + - job: UT_On_Ubuntu + variables: + group: AZCOPY_TESTS_VAR + isMutexSet: 'false' + # allow maximum build time, in case we have build congestion + timeoutInMinutes: 360 + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + name: 'Set_up_Python' + inputs: + versionSpec: '3.7' + + - task: GoTool@0 + name: 'Set_up_Golang' + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + - task: DownloadSecureFile@1 + name: ciGCSServiceAccountKey + displayName: 'Download GCS Service Account Key' + inputs: + secureFile: 'ci-gcs-dev.json' + + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Install dependencies' + + - script: | + pip install azure-storage-blob==12.12.0 + # set the variable to indicate that the mutex is being acquired + # note: we set it before acquiring the mutex to ensure we release the mutex. + # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # mutex but before we set this variable. + # setting this before will always work since it is valid to break an un-acquired mutex. + echo '##vso[task.setvariable variable=isMutexSet]true' + # acquire the mutex before running live tests to avoid conflicts + python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + name: 'Acquire_the_distributed_mutex' + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'cmd' + coverage_name: 'cmd' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common' + coverage_name: 'common' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'common/parallel' + coverage_name: 'parallel' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'ste' + coverage_name: 'ste' + + - template: azurePipelineTemplates/run-ut.yml + parameters: + directory: 'sddl' + coverage_name: 'sddl' + + - script: | + GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + go build -tags "netgo" -o test-validator ./testSuite/ + mkdir test-temp + mkdir coverage + export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + keyctl session test python ./testSuite/scripts/run.py - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) - # variables: - # - group: AZCOPY_TESTS_VAR - - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o smoke_coverage.txt + echo 'Formatting coverage to json format' + $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + echo 'Formatting coverage to xml format' + $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + name: 'Run_smoke_tests' + env: + ACCOUNT_NAME: $(ACCOUNT_NAME) + ACCOUNT_KEY: $(ACCOUNT_KEY) + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + FILESYSTEM_URL: $(FILESYSTEM_URL) + OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + SHARE_SAS_URL: $(SHARE_SAS_URL) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + condition: succeededOrFailed() + # Smoke Tests Publishing + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + - script: | + pip install azure-storage-blob==12.12.0 + python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + name: 'Release_the_distributed_mutex' + # this runs even if the job was canceled (only if the mutex was acquired by this job) + condition: and(always(), eq(variables['isMutexSet'], 'true')) + - job: E2E_Test + timeoutInMinutes: 360 + # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + build_name: 'azcopy_linux_amd64' + display_name: "Linux" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + Windows: + imageName: 'windows-2019' + build_name: 'azcopy_windows_amd64.exe' + display_name: "Windows" + type: 'windows' + go_path: 'C:\Users\VssAdministrator\go\bin\' + suffix: '.exe' + run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + output_test_logs: 'cat test.txt' + save_exit_code: 'set exit_code=%errorlevel%' + return_exit_code: 'exit %exit_code%' + go_build: 'go build -cover -o $(build_name)' + MacOS: + imageName: 'macos-11' + build_name: 'azcopy_darwin_amd64' + display_name: "MacOS" + go_path: '$(go env GOPATH)/bin/' + suffix: '' + run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + output_test_logs: '' + save_exit_code: 'exit_code=${PIPESTATUS[0]}' + return_exit_code: 'exit "$exit_code"' + go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + pool: + vmImage: $(imageName) + variables: + - group: AZCOPY_TESTS_VAR + + steps: + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + pwsh: 'true' + displayName: 'Install Powershell Az Module' + - task: GoTool@0 + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Installing dependencies' + - bash: | + echo "##vso[task.setvariable variable=CGO_ENABLED]0" + displayName: 'Set CGO_ENABLED for Windows' + condition: eq(variables.type, 'windows') + - bash: | + npm install -g azurite + mkdir azurite + azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + # Running E2E Tests on AMD64 + - script: | + mkdir coverage + echo 'Building executable' + $(go_build) + echo 'Running tests' + $(run_e2e) + $(save_exit_code) + echo "Generating junit report" + cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + echo 'Formatting coverage directory to legacy txt format' + go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + $(output_test_logs) + echo 'Formatting coverage to json format' + $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + echo 'Formatting coverage to xml format' + $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + $(return_exit_code) + env: + AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + displayName: 'E2E Test $(display_name) - AMD64' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish logs' + condition: succeededOrFailed() + inputs: + pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + artifactName: logs + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + testRunTitle: 'Go on $(display_name)' + + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts #dependsOn: RunTests From 1e6bca42d6dd6dec810788e8286c38d87cb9fafa Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 21:09:08 +0530 Subject: [PATCH 284/513] test --- release-pipelines.yml | 514 +++++++++++++++++++-------------------- setup/test_artifacts.yml | 10 - 2 files changed, 257 insertions(+), 267 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index cf8178083..14117a511 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -50,264 +50,264 @@ trigger: none pr: none stages: - - stage: RunTests - jobs: - - job: UT_On_Ubuntu - variables: - group: AZCOPY_TESTS_VAR - isMutexSet: 'false' - # allow maximum build time, in case we have build congestion - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - name: 'Set_up_Python' - inputs: - versionSpec: '3.7' - - - task: GoTool@0 - name: 'Set_up_Golang' - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - - task: DownloadSecureFile@1 - name: ciGCSServiceAccountKey - displayName: 'Download GCS Service Account Key' - inputs: - secureFile: 'ci-gcs-dev.json' - - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Install dependencies' - - - script: | - pip install azure-storage-blob==12.12.0 - # set the variable to indicate that the mutex is being acquired - # note: we set it before acquiring the mutex to ensure we release the mutex. - # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # mutex but before we set this variable. - # setting this before will always work since it is valid to break an un-acquired mutex. - echo '##vso[task.setvariable variable=isMutexSet]true' - # acquire the mutex before running live tests to avoid conflicts - python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - name: 'Acquire_the_distributed_mutex' - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'cmd' - coverage_name: 'cmd' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common' - coverage_name: 'common' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'common/parallel' - coverage_name: 'parallel' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'ste' - coverage_name: 'ste' - - - template: azurePipelineTemplates/run-ut.yml - parameters: - directory: 'sddl' - coverage_name: 'sddl' - - - script: | - GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - go build -tags "netgo" -o test-validator ./testSuite/ - mkdir test-temp - mkdir coverage - export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - keyctl session test python ./testSuite/scripts/run.py + # - stage: RunTests + # jobs: + # - job: UT_On_Ubuntu + # variables: + # group: AZCOPY_TESTS_VAR + # isMutexSet: 'false' + # # allow maximum build time, in case we have build congestion + # timeoutInMinutes: 360 + # pool: + # vmImage: 'ubuntu-20.04' + # steps: + # - task: UsePythonVersion@0 + # name: 'Set_up_Python' + # inputs: + # versionSpec: '3.7' + + # - task: GoTool@0 + # name: 'Set_up_Golang' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + # - task: DownloadSecureFile@1 + # name: ciGCSServiceAccountKey + # displayName: 'Download GCS Service Account Key' + # inputs: + # secureFile: 'ci-gcs-dev.json' + + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Install dependencies' + + # - script: | + # pip install azure-storage-blob==12.12.0 + # # set the variable to indicate that the mutex is being acquired + # # note: we set it before acquiring the mutex to ensure we release the mutex. + # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # # mutex but before we set this variable. + # # setting this before will always work since it is valid to break an un-acquired mutex. + # echo '##vso[task.setvariable variable=isMutexSet]true' + # # acquire the mutex before running live tests to avoid conflicts + # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + # name: 'Acquire_the_distributed_mutex' + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'cmd' + # coverage_name: 'cmd' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common' + # coverage_name: 'common' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common/parallel' + # coverage_name: 'parallel' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'ste' + # coverage_name: 'ste' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'sddl' + # coverage_name: 'sddl' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + # go build -tags "netgo" -o test-validator ./testSuite/ + # mkdir test-temp + # mkdir coverage + # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + # export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + # keyctl session test python ./testSuite/scripts/run.py - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o smoke_coverage.txt - echo 'Formatting coverage to json format' - $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - echo 'Formatting coverage to xml format' - $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - name: 'Run_smoke_tests' - env: - ACCOUNT_NAME: $(ACCOUNT_NAME) - ACCOUNT_KEY: $(ACCOUNT_KEY) - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - FILESYSTEM_URL: $(FILESYSTEM_URL) - OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - SHARE_SAS_URL: $(SHARE_SAS_URL) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - condition: succeededOrFailed() - # Smoke Tests Publishing - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - - script: | - pip install azure-storage-blob==12.12.0 - python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - name: 'Release_the_distributed_mutex' - # this runs even if the job was canceled (only if the mutex was acquired by this job) - condition: and(always(), eq(variables['isMutexSet'], 'true')) - - job: E2E_Test - timeoutInMinutes: 360 - # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - strategy: - matrix: - Ubuntu-20: - imageName: 'ubuntu-20.04' - build_name: 'azcopy_linux_amd64' - display_name: "Linux" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - Windows: - imageName: 'windows-2019' - build_name: 'azcopy_windows_amd64.exe' - display_name: "Windows" - type: 'windows' - go_path: 'C:\Users\VssAdministrator\go\bin\' - suffix: '.exe' - run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - output_test_logs: 'cat test.txt' - save_exit_code: 'set exit_code=%errorlevel%' - return_exit_code: 'exit %exit_code%' - go_build: 'go build -cover -o $(build_name)' - MacOS: - imageName: 'macos-11' - build_name: 'azcopy_darwin_amd64' - display_name: "MacOS" - go_path: '$(go env GOPATH)/bin/' - suffix: '' - run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - output_test_logs: '' - save_exit_code: 'exit_code=${PIPESTATUS[0]}' - return_exit_code: 'exit "$exit_code"' - go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - pool: - vmImage: $(imageName) - variables: - - group: AZCOPY_TESTS_VAR - - steps: - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - pwsh: 'true' - displayName: 'Install Powershell Az Module' - - task: GoTool@0 - inputs: - version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - script: | - go install github.com/jstemmer/go-junit-report@v0.9.1 - go install github.com/axw/gocov/gocov@v1.1.0 - go install github.com/AlekSi/gocov-xml@v1.0.0 - go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - displayName: 'Installing dependencies' - - bash: | - echo "##vso[task.setvariable variable=CGO_ENABLED]0" - displayName: 'Set CGO_ENABLED for Windows' - condition: eq(variables.type, 'windows') - - bash: | - npm install -g azurite - mkdir azurite - azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' - # Running E2E Tests on AMD64 - - script: | - mkdir coverage - echo 'Building executable' - $(go_build) - echo 'Running tests' - $(run_e2e) - $(save_exit_code) - echo "Generating junit report" - cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - echo 'Formatting coverage directory to legacy txt format' - go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - $(output_test_logs) - echo 'Formatting coverage to json format' - $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - echo 'Formatting coverage to xml format' - $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - $(return_exit_code) - env: - AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - displayName: 'E2E Test $(display_name) - AMD64' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish logs' - condition: succeededOrFailed() - inputs: - pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - artifactName: logs - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testRunner: JUnit - testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - testRunTitle: 'Go on $(display_name)' - - - task: PublishCodeCoverageResults@1 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o smoke_coverage.txt + # echo 'Formatting coverage to json format' + # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + # echo 'Formatting coverage to xml format' + # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + # name: 'Run_smoke_tests' + # env: + # ACCOUNT_NAME: $(ACCOUNT_NAME) + # ACCOUNT_KEY: $(ACCOUNT_KEY) + # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + # FILESYSTEM_URL: $(FILESYSTEM_URL) + # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + # SHARE_SAS_URL: $(SHARE_SAS_URL) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # condition: succeededOrFailed() + # # Smoke Tests Publishing + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + # - script: | + # pip install azure-storage-blob==12.12.0 + # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + # name: 'Release_the_distributed_mutex' + # # this runs even if the job was canceled (only if the mutex was acquired by this job) + # condition: and(always(), eq(variables['isMutexSet'], 'true')) + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) + # variables: + # - group: AZCOPY_TESTS_VAR + + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' + + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs + + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' + + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - stage: BuildArtifacts #dependsOn: RunTests diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 528e0fc45..8525605f3 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -7,16 +7,6 @@ parameters: type: string steps: - #TODO: Remove this while merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # displayName: 'List files after downloading go_installer.sh' - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy - task: ShellScript@2 inputs: From 5c43125d183f2006e6679ee7664082b4429206b2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 22:23:52 +0530 Subject: [PATCH 285/513] test --- release-pipelines.yml | 5 +++++ setup/test_artifacts.yml | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 14117a511..1344dc0ef 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1294,6 +1294,7 @@ stages: pool: vmImage: $(vmImage) steps: + - checkout: none - script: | sudo apt update sudo apt --fix-broken install @@ -1335,6 +1336,7 @@ stages: - ImageOverride -equals $(AgentName) steps: + - checkout: none - script: | sudo apt update sudo apt --fix-broken install @@ -1376,6 +1378,7 @@ stages: - ImageOverride -equals $(agentName) steps: + - checkout: none - script: | sudo yum update -y sudo yum install git -y @@ -1418,6 +1421,7 @@ stages: - ImageOverride -equals $(agentName) steps: + - checkout: none - script: | sudo yum update -y sudo yum install git -y @@ -1466,6 +1470,7 @@ stages: - ImageOverride -equals $(AgentName) steps: + - checkout: none - script: | sudo tdnf update -y sudo tdnf install git -y diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 8525605f3..f09b47044 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -7,6 +7,15 @@ parameters: type: string steps: + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy - task: ShellScript@2 inputs: From c0e578a89ea878c27b87521ea46f85fa76194cfd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 18 Jun 2024 22:53:06 +0530 Subject: [PATCH 286/513] test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1344dc0ef..1ed166cd0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1478,7 +1478,7 @@ stages: - template: setup/test_artifacts.yml parameters: - artifact_name: 'azCopy-linux-signed/mariner' + artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' From 7367702b750d5df4e6bc9622a557e4af06dab1e4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 10:12:51 +0530 Subject: [PATCH 287/513] test --- release-pipelines.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1ed166cd0..233000e37 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1723,10 +1723,13 @@ stages: # Find the current version raw_version_output=$(azcopy_linux_amd64 --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') - + + echo "Test Mode value $(Test_Mode)" + ls -ltR $(Build.ArtifactStagingDirectory) + today=$(date +"%Y%m%d") - if [ "$(Test_Mode)" = "true" ]; then + if [ $(Test_Mode) = "true" ]; then container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" else container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" @@ -1896,7 +1899,9 @@ stages: displayName: 'List Artifacts' - script: | + mv azCopy-linux-signed/*tar.gz . pmc package upload azCopy-linux-signed + mv *tar.gz azCopy-linux-signed/*tar.gz displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1958,6 +1963,16 @@ stages: condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + pool: vmImage: 'ubuntu-20.04' variables: @@ -1984,19 +1999,20 @@ stages: # install azcopy - script: | cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - ls azcopy*.arm64.rpm - sudo apt-get install ./`ls azcopy*.arm64.rpm` -y + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb azcopy --version displayName: 'Installing azcopy' - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-storage/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py + wget https://raw.githubusercontent.com/Azure/azure-storage-storage/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py ls -l displayName: 'Getting Python script' workingDirectory: $(root_dir) - script: | - python3 setup/releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) From 0ee8fca638b2604a6a2ce1cd31b871230cdd2fd6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 10:13:46 +0530 Subject: [PATCH 288/513] test --- release-pipelines.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 233000e37..29c2a48b7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1962,17 +1962,7 @@ stages: dependsOn: ReleaseArtifacts condition: succeeded('ReleaseArtifacts') jobs: - - job: UpdateVersion - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - + - job: UpdateVersion pool: vmImage: 'ubuntu-20.04' variables: From 4b58b9dcdbe897307a3e78f175e1b4b0bd9c266a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 11:03:24 +0530 Subject: [PATCH 289/513] test --- release-pipelines.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 29c2a48b7..1c8be9e2c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1900,11 +1900,12 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . - pmc package upload azCopy-linux-signed - mv *tar.gz azCopy-linux-signed/*tar.gz + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + mv *.tar.gz azCopy-linux-signed/ displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) + # replace org name while merging to main - script: | wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv cat ./packages.csv @@ -1994,9 +1995,10 @@ stages: sudo dpkg -i azcopy-*arm64.deb azcopy --version displayName: 'Installing azcopy' - + + # TODO: change org name while merging to main - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-storage/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py ls -l displayName: 'Getting Python script' workingDirectory: $(root_dir) From f1ee69075e6811a96093267889bacede3c3bba39 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 11:45:53 +0530 Subject: [PATCH 290/513] test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1c8be9e2c..b30f8f074 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1901,7 +1901,6 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed - mv *.tar.gz azCopy-linux-signed/ displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -2004,6 +2003,7 @@ stages: workingDirectory: $(root_dir) - script: | + pip install azure-storage-blob python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) From fb8f693e20cff7e27ced451e4b6744de7ef1f2a0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 11:50:09 +0530 Subject: [PATCH 291/513] test --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b30f8f074..851856bca 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1729,7 +1729,7 @@ stages: today=$(date +"%Y%m%d") - if [ $(Test_Mode) = "true" ]; then + if [ $(Test_Mode) = "True" ]; then container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" else container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" @@ -1813,8 +1813,8 @@ stages: target: '$(Build.SourceVersion)' tagSource: 'userSpecifiedTag' - title: 'azcopy-10.25.0' - tag: 'azcopy-10.25.0' + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} assets: | $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* From 89c4971c3d5c000bbd1c46b7b19200a4370a0c71 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 13:38:18 +0530 Subject: [PATCH 292/513] test --- packages.csv | 46 +++++++++++++++++++++---------------------- release-pipelines.yml | 34 +++++++++++++------------------- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/packages.csv b/packages.csv index 4b578b5eb..8d03e6d48 100644 --- a/packages.csv +++ b/packages.csv @@ -1,26 +1,26 @@ # Do not remove these comments # Format of the file : ,,, -Ubuntu-18.04,fuse2AmdDeb,microsoft-ubuntu-bionic-prod-apt,bionic -Ubuntu-20.04,fuse3AmdDeb,microsoft-ubuntu-focal-prod-apt,focal -Ubuntu-22.04,fuse3AmdDeb,microsoft-ubuntu-jammy-prod-apt,jammy -Ubuntu-22.04,fuse3ArmDeb,microsoft-ubuntu-jammy-prod-apt,jammy -Ubuntu-24.04,fuse3AmdDeb,microsoft-ubuntu-noble-prod-apt,noble -Ubuntu-24.04,fuse3ArmDeb,microsoft-ubuntu-noble-prod-apt,noble -Debian-9.0,fuse2AmdDeb,microsoft-debian-stretch-prod-apt,stretch -Debian-10.0,fuse2AmdDeb,microsoft-debian-buster-prod-apt,buster +Ubuntu-18.04,azcopyAmdDeb,microsoft-ubuntu-bionic-prod-apt,bionic +Ubuntu-20.04,azcopyAmdDeb,microsoft-ubuntu-focal-prod-apt,focal +Ubuntu-22.04,azcopyAmdDeb,microsoft-ubuntu-jammy-prod-apt,jammy +Ubuntu-22.04,azcopyArmDeb,microsoft-ubuntu-jammy-prod-apt,jammy +Ubuntu-24.04,azcopyAmdDeb,microsoft-ubuntu-noble-prod-apt,noble +Ubuntu-24.04,azcopyArmDeb,microsoft-ubuntu-noble-prod-apt,noble +Debian-9.0,azcopyAmdDeb,microsoft-debian-stretch-prod-apt,stretch +Debian-10.0,azcopyAmdDeb,microsoft-debian-buster-prod-apt,buster Debian-11.0,fuse3AmdDeb,microsoft-debian-bullseye-prod-apt,bullseye -RHEL-7.5,fuse3AmdRpm,microsoft-rhel7.5-prod-yum, -RHEL-7.8,fuse3AmdRpm,microsoft-rhel7.8-prod-yum, -RHEL-8.1,fuse3AmdRpm,microsoft-rhel8.1-prod-yum, -RHEL-8.2,fuse3AmdRpm,microsoft-rhel8.2-prod-yum, -RHEL-7.0,fuse3AmdRpm,microsoft-rhel7.0-prod-yum, -RHEL-8.0,fuse3AmdRpm,microsoft-rhel8.0-prod-yum, -RHEL-9.0,fuse3AmdRpm,microsoft-rhel9.0-prod-yum, -RHEL-9.0,fuse3ArmRpm,microsoft-rhel9.0-prod-yum, -CentOS-7.0,fuse3AmdRpm,microsoft-centos7-prod-yum, -CentOS-8.0,fuse3AmdRpm,microsoft-centos8-prod-yum, -SUSE-15Gen2,fuse3AmdRpm,microsoft-sles15-prod-yum, -Mariner-2.0-x86_64,marinerFuse3AmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerFuse3AarchRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, -Rocky-8.0,fuse3AmdRpm,microsoft-el8-prod-yum, -Rocky-9.0,fuse3AmdRpm,microsoft-el9-prod-yum, \ No newline at end of file +RHEL-7.5,azcopyAmdRpm,microsoft-rhel7.5-prod-yum, +RHEL-7.8,azcopyAmdRpm,microsoft-rhel7.8-prod-yum, +RHEL-8.1,azcopyAmdRpm,microsoft-rhel8.1-prod-yum, +RHEL-8.2,azcopyAmdRpm,microsoft-rhel8.2-prod-yum, +RHEL-7.0,azcopyAmdRpm,microsoft-rhel7.0-prod-yum, +RHEL-8.0,azcopyAmdRpm,microsoft-rhel8.0-prod-yum, +RHEL-9.0,azcopyAmdRpm,microsoft-rhel9.0-prod-yum, +RHEL-9.0,azcopyArmRpm,microsoft-rhel9.0-prod-yum, +CentOS-7.0,azcopyAmdRpm,microsoft-centos7-prod-yum, +CentOS-8.0,azcopyAmdRpm,microsoft-centos8-prod-yum, +SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, +Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, +Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, +Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, +Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, \ No newline at end of file diff --git a/release-pipelines.yml b/release-pipelines.yml index 851856bca..35a78102b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1912,31 +1912,25 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - for file in "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed"/*; - do - if [ -f "$file" ]; then - fileID=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $file` - echo "File: $file and File ID: $fileID" - fi - done - - # marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.arm64.rpm` - # echo "Azcopy mariner AMD DEB ID: $marinerArmRpm" + marinerArmRpmFile=$(ls azcopy* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile` + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - # marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` - # echo "Azcopy mariner AMD DEB ID: $marinerAmdRpm" + marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - # azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - # echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - # azcopyAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.rpm` - # echo "Azcopy AMD DEB ID: $azcopyAmdRpm" + azcopyAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.rpm` + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - # echo "Azcopy AMD DEB ID: $azcopyArmDeb" + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" - # azcopyArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.rpm` - # echo "Azcopy AMD DEB ID: $azcopyArmRpm" + azcopyArmRpmFile=$(ls azcopy* | grep -v 'cm2\.arm64\.rpm') + azcopyArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile` + echo "Azcopy ARM RPM ID: $azcopyArmRpm" while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" From 3e4a36a14fac62e7bce850f76629ae9e02dac7f3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 14:50:12 +0530 Subject: [PATCH 293/513] test --- release-pipelines.yml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 35a78102b..25f64c1a6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1912,24 +1912,36 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - marinerArmRpmFile=$(ls azcopy* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile` - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + # Find the Mariner RPM file and upload + marinerArmRpmFile=$(ls $(root_dir)/azcopy-* | grep 'cm2\.arm64\.rpm') + if [ -n "$marinerArmRpmFile" ]; then + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + else + echo "No Mariner ARM RPM file found." + fi - marinerAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*-cm2.x86_64.rpm` + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls $(root_dir)/azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + marinerArmRpmFile=$(ls $(root_dir)/azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - azcopyAmdRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.rpm` - echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - + azcopyAmdRpmFile=$(ls $(root_dir)/azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` echo "Azcopy ARM DEB ID: $azcopyArmDeb" - azcopyArmRpmFile=$(ls azcopy* | grep -v 'cm2\.arm64\.rpm') - azcopyArmRpm=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile` + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) echo "Azcopy ARM RPM ID: $azcopyArmRpm" while IFS=, read -r distro fuseArchType repoName releaseName; do From 27b227428f3ce332b601d38907ad35c69b0a5f65 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 15:16:02 +0530 Subject: [PATCH 294/513] test --- setup/releaseVersionUpdate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index f00c2f57e..7de6f6e68 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -18,6 +18,7 @@ # delete latest version file in the container deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +print(f"deleteURL: '{deleteUrl}'") resp = requests.delete(deleteUrl) sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') From f8af67cf5610241a7e4ab8b10e3bbd10e0be6c47 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 19 Jun 2024 17:59:27 +0530 Subject: [PATCH 295/513] test --- release-pipelines.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 25f64c1a6..0642c0fd3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1913,27 +1913,23 @@ stages: - script: | # Find the Mariner RPM file and upload - marinerArmRpmFile=$(ls $(root_dir)/azcopy-* | grep 'cm2\.arm64\.rpm') - if [ -n "$marinerArmRpmFile" ]; then - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - else - echo "No Mariner ARM RPM file found." - fi + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls $(root_dir)/azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - marinerArmRpmFile=$(ls $(root_dir)/azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - azcopyAmdRpmFile=$(ls $(root_dir)/azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" From 64cfd626e6a8b0171116c4c902fcc5c4fea4c325 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 20 Jun 2024 10:41:15 +0530 Subject: [PATCH 296/513] test --- release-pipelines.yml | 51 +++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0642c0fd3..a908588b6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1153,10 +1153,10 @@ stages: sudo apt-get clean sudo apt-get update sudo apt-get upgrade -y + sudo apt-get install unzip -y sudo apt-get install -f --fix-missing wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb - sudo apt update sudo apt install apt-transport-https -y sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" @@ -1260,6 +1260,7 @@ stages: # Validate signed images have md5sum changed - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ @@ -1315,12 +1316,9 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_2 timeoutInMinutes: 120 @@ -1356,12 +1354,9 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_arm64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_arm64 --help - displayName: 'Check Help' + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_3 timeoutInMinutes: 120 @@ -1399,12 +1394,9 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_arm64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_arm64 --help - displayName: 'Check Help' + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_4 timeoutInMinutes: 120 @@ -1447,12 +1439,9 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_5 timeoutInMinutes: 120 @@ -1494,12 +1483,9 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | - azcopy_linux_amd64 --version - displayName: 'Check Version' - - - script: | - azcopy_linux_amd64 --help - displayName: 'Check Help' + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_6 timeoutInMinutes: 360 @@ -1724,7 +1710,6 @@ stages: raw_version_output=$(azcopy_linux_amd64 --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') - echo "Test Mode value $(Test_Mode)" ls -ltR $(Build.ArtifactStagingDirectory) today=$(date +"%Y%m%d") @@ -1742,7 +1727,7 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/drop.zip" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - job: ReleaseAzcopy timeoutInMinutes: 120 From eb0bc08e042f69664095c1c3d8d120658d35acf1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 20 Jun 2024 10:51:04 +0530 Subject: [PATCH 297/513] test --- setup/releaseVersionUpdate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index 7de6f6e68..b70c0474e 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -17,10 +17,10 @@ sasToken = sasUrl.split('?')[1] # delete latest version file in the container -deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken -print(f"deleteURL: '{deleteUrl}'") -resp = requests.delete(deleteUrl) -sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') +# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +# print(f"deleteURL: '{deleteUrl}'") +# resp = requests.delete(deleteUrl) +# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') # Create a file and write data to it file_name = 'latest_version.txt' From 2638db4d61ed0afa1890fc6618e35b913ef52895 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 20 Jun 2024 11:01:14 +0530 Subject: [PATCH 298/513] Release pipeline for linux, mariner, mac and windows --- go_installer.sh | 14 + packages.csv | 14 + release-pipeline.yml | 1995 ++++++++++++++++++++++++++++ setup/esrp_sign_linux.yml | 25 + setup/esrp_sign_notarize_macos.yml | 59 + setup/esrp_sign_windows.yml | 50 + setup/releaseVersionUpdate.py | 50 + setup/test_artifacts.yml | 40 + 8 files changed, 2247 insertions(+) create mode 100644 go_installer.sh create mode 100644 packages.csv create mode 100644 release-pipeline.yml create mode 100644 setup/esrp_sign_linux.yml create mode 100644 setup/esrp_sign_notarize_macos.yml create mode 100644 setup/esrp_sign_windows.yml create mode 100644 setup/releaseVersionUpdate.py create mode 100644 setup/test_artifacts.yml diff --git a/go_installer.sh b/go_installer.sh new file mode 100644 index 000000000..932951471 --- /dev/null +++ b/go_installer.sh @@ -0,0 +1,14 @@ +#!/bin/bash +work_dir=$(echo $1 | sed 's:/*$::') +version="1.21.8" +arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` + +if [ $arch != "arm64" ];then + arch="amd64" +fi + +echo "Installing on : " $arch " Version : " $version +wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir" +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz +sudo ln -sf /usr/local/go/bin/go /usr/bin/go \ No newline at end of file diff --git a/packages.csv b/packages.csv new file mode 100644 index 000000000..932951471 --- /dev/null +++ b/packages.csv @@ -0,0 +1,14 @@ +#!/bin/bash +work_dir=$(echo $1 | sed 's:/*$::') +version="1.21.8" +arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` + +if [ $arch != "arm64" ];then + arch="amd64" +fi + +echo "Installing on : " $arch " Version : " $version +wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir" +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz +sudo ln -sf /usr/local/go/bin/go /usr/bin/go \ No newline at end of file diff --git a/release-pipeline.yml b/release-pipeline.yml new file mode 100644 index 000000000..2fa11f77f --- /dev/null +++ b/release-pipeline.yml @@ -0,0 +1,1995 @@ +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' + +parameters: + - name: tag + displayName: 'New Release Tag' + type: string + default: 'azcopy-' + + - name: test_mode + displayName: 'Test Mode' + type: boolean + default: false + + # - name: unit_test + # displayName: 'Execute Unit Tests' + # type: boolean + # default: false + + - name: post_release + displayName: 'Post Release on Github' + type: boolean + default: false + + - name: publish_artifacts + displayName: 'Publish Artifacts to Linux Repos' + type: boolean + default: false + + - name: draft + displayName: 'Post as Draft Release' + type: boolean + default: false + + - name: prerelease + displayName: 'Post as PreRelease' + type: boolean + default: false + + - name: update_version + displayName: 'Update Version' + type: boolean + default: false + +# Do not trigger this pipeline automatically +trigger: none +pr: none + +stages: + # - stage: RunTests + # jobs: + # - job: UT_On_Ubuntu + # variables: + # group: AZCOPY_TESTS_VAR + # isMutexSet: 'false' + # # allow maximum build time, in case we have build congestion + # timeoutInMinutes: 360 + # pool: + # vmImage: 'ubuntu-20.04' + # steps: + # - task: UsePythonVersion@0 + # name: 'Set_up_Python' + # inputs: + # versionSpec: '3.7' + + # - task: GoTool@0 + # name: 'Set_up_Golang' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + # - task: DownloadSecureFile@1 + # name: ciGCSServiceAccountKey + # displayName: 'Download GCS Service Account Key' + # inputs: + # secureFile: 'ci-gcs-dev.json' + + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Install dependencies' + + # - script: | + # pip install azure-storage-blob==12.12.0 + # # set the variable to indicate that the mutex is being acquired + # # note: we set it before acquiring the mutex to ensure we release the mutex. + # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # # mutex but before we set this variable. + # # setting this before will always work since it is valid to break an un-acquired mutex. + # echo '##vso[task.setvariable variable=isMutexSet]true' + # # acquire the mutex before running live tests to avoid conflicts + # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + # name: 'Acquire_the_distributed_mutex' + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'cmd' + # coverage_name: 'cmd' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common' + # coverage_name: 'common' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common/parallel' + # coverage_name: 'parallel' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'ste' + # coverage_name: 'ste' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'sddl' + # coverage_name: 'sddl' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + # go build -tags "netgo" -o test-validator ./testSuite/ + # mkdir test-temp + # mkdir coverage + # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + # export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + # keyctl session test python ./testSuite/scripts/run.py + + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o smoke_coverage.txt + # echo 'Formatting coverage to json format' + # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + # echo 'Formatting coverage to xml format' + # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + # name: 'Run_smoke_tests' + # env: + # ACCOUNT_NAME: $(ACCOUNT_NAME) + # ACCOUNT_KEY: $(ACCOUNT_KEY) + # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + # FILESYSTEM_URL: $(FILESYSTEM_URL) + # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + # SHARE_SAS_URL: $(SHARE_SAS_URL) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # condition: succeededOrFailed() + # # Smoke Tests Publishing + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + # - script: | + # pip install azure-storage-blob==12.12.0 + # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + # name: 'Release_the_distributed_mutex' + # # this runs even if the job was canceled (only if the mutex was acquired by this job) + # condition: and(always(), eq(variables['isMutexSet'], 'true')) + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) + # variables: + # - group: AZCOPY_TESTS_VAR + + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' + + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs + + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' + + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + + - stage: BuildArtifacts + #dependsOn: RunTests + #condition: succeeded('RunTests') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files for packaging' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy_linux_amd64 --version + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + displayName: 'Copy required files' + + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy_linux_arm64 --version + azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here + + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)/azcopy_windows_amd64' + - name: windows_arm64 + value: '$(work_dir)/azcopy_windows_arm64' + - name: windows_386 + value: '$(work_dir)/azcopy_windows_386' + - name: archives + value: '$(work_dir)/archives' + + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir -p $(windows_amd64) + mkdir -p $(windows_arm64) + mkdir -p $(windows_386) + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: '*.exe' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped + unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped + mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/unzipped + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: '*.exe' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' + + # SignArtifacts end here + + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' + + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command' + + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Verify that azcopy.exe exists + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { + Write-Error "azcopy.exe not found in extraction directory: $extractDir" + exit 1 + } + + # Run the azcopy version command + .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version + displayName: 'Extract Files and Run Version Command on Windows' + + # TestArtifacts ends here + + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToContainer + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + - name: Test_Mode + value: ${{ parameters.test_mode }} + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Binaries Build Artifacts' + inputs: + artifactName: 'azCopy-binaries-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get install -y zip + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Create drop.zip file and install azcopy deb package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - task: ArchiveFiles@2 + displayName: 'Archive all binaries to drop.zip' + inputs: + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' + archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' + continueOnError: true + + - task: Bash@3 + displayName: 'Install Azure CLI' + inputs: + targetType: 'inline' + script: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=$(azcopy_linux_amd64 --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + ls -ltR $(Build.ArtifactStagingDirectory) + + today=$(date +"%Y%m%d") + + if [ $(Test_Mode) = "True" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy_linux_amd64 + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - ${{ if eq(parameters.post_release, true) }}: + + # add release tags & push to github + #TODO: update github connection and repo name while merging to main + - task: GithubRelease@1 + inputs: + githubConnection: 'dphulkar-msft-github-rel' + repositoryName: 'dphulkar-msft/azure-storage-azcopy' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + mv azCopy-linux-signed/*tar.gz . + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + displayName: 'Upload files' + workingDirectory: $(Build.ArtifactStagingDirectory) + + # replace org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + # Find the Mariner RPM file and upload + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - stage: UpdateLatestVersion + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - ${{ if eq(parameters.update_version, true) }}: + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' + + # TODO: change org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) + diff --git a/setup/esrp_sign_linux.yml b/setup/esrp_sign_linux.yml new file mode 100644 index 000000000..949238cc0 --- /dev/null +++ b/setup/esrp_sign_linux.yml @@ -0,0 +1,25 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_KEY_CODE: '' + Pattern: '' + DisplayName: '' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ${{ parameters.DisplayName }} + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "${{ parameters.ESRP_AZCOPY_KEY_CODE }}", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] diff --git a/setup/esrp_sign_notarize_macos.yml b/setup/esrp_sign_notarize_macos.yml new file mode 100644 index 000000000..fb4d4ff1f --- /dev/null +++ b/setup/esrp_sign_notarize_macos.yml @@ -0,0 +1,59 @@ +# esrp-sign-notarize-macos.yml +parameters: + FolderPath: '' + ESRP_AZCOPY_MAC_KEY_CODE: '' + Pattern: '*.zip' + + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Notarize Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 \ No newline at end of file diff --git a/setup/esrp_sign_windows.yml b/setup/esrp_sign_windows.yml new file mode 100644 index 000000000..1aeb8ee68 --- /dev/null +++ b/setup/esrp_sign_windows.yml @@ -0,0 +1,50 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_WIN_KEY_CODE: '' + Pattern: '*.exe' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "${{ parameters.ESRP_AZCOPY_WIN_KEY_CODE }}", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py new file mode 100644 index 000000000..b70c0474e --- /dev/null +++ b/setup/releaseVersionUpdate.py @@ -0,0 +1,50 @@ +import json +import requests +import sys +import os +from xml.dom import minidom +from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient + + +sasUrl = sys.argv[1] +releaseVersion = sys.argv[2].split(' ')[2] +print('Release Version: ' + releaseVersion) +if(len(releaseVersion)==0): + print('Incorrect Release Version') + sys.exit(1) + +containerUrl = sasUrl.split('?')[0] +sasToken = sasUrl.split('?')[1] + +# delete latest version file in the container +# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +# print(f"deleteURL: '{deleteUrl}'") +# resp = requests.delete(deleteUrl) +# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') + +# Create a file and write data to it +file_name = 'latest_version.txt' + +# Open the file in write mode +with open(file_name, 'w') as file: + file.write(releaseVersion) +print(f'Data written to {file_name}') + +# Create the BlobServiceClient using the SAS URL +blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) + +# Get the container name from the SAS URL +container_name = sasUrl.split('?')[0].split('/')[-1] + +# Get a client to interact with the container +container_client = blob_service_client.get_container_client(container_name) + +# Upload the file to the container +with open(file_name, 'rb') as data: + container_client.upload_blob(name=file_name, data=data, overwrite=True) + +print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + +# Clean up the local file +os.remove(file_name) +print(f"Local file '{file_name}' deleted.") diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml new file mode 100644 index 000000000..f09b47044 --- /dev/null +++ b/setup/test_artifacts.yml @@ -0,0 +1,40 @@ +parameters: + - name: artifact_name + type: string + - name: download_path + type: string + - name: item_pattern + type: string + +steps: + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + + - task: ShellScript@2 + inputs: + scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" + args: "$(System.DefaultWorkingDirectory)/" + displayName: "GoTool Custom Setup" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: ${{ parameters.artifact_name }} + downloadPath: ${{ parameters.download_path }} + itemPattern: ${{ parameters.item_pattern }} + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} From c21cc9b2009f60759649d9529bf682f1bb801b31 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 20 Jun 2024 11:25:51 +0530 Subject: [PATCH 299/513] Release pipeline for linux, mariner, mac and windows --- release-pipelines.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a908588b6..dff49bb23 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1316,8 +1316,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help displayName: 'Check Version and Help' - job: Set_2 @@ -1354,8 +1354,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_arm64 --version + azcopy_linux_arm64 --help displayName: 'Check Version and Help' - job: Set_3 @@ -1394,8 +1394,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_arm64 --version + azcopy_linux_arm64 --help displayName: 'Check Version and Help' - job: Set_4 @@ -1439,8 +1439,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help displayName: 'Check Version and Help' - job: Set_5 @@ -1483,8 +1483,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | - azcopy --version - azcopy --help + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help displayName: 'Check Version and Help' - job: Set_6 From 5c9f872f036c294adc3e826441f98976537b95e2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 20 Jun 2024 11:01:14 +0530 Subject: [PATCH 300/513] Release pipeline for linux, mariner, mac and windows --- go_installer.sh | 14 + packages.csv | 14 + release-pipeline.yml | 1996 ++++++++++++++++++++++++++++ setup/esrp_sign_linux.yml | 25 + setup/esrp_sign_notarize_macos.yml | 59 + setup/esrp_sign_windows.yml | 50 + setup/releaseVersionUpdate.py | 50 + setup/test_artifacts.yml | 40 + 8 files changed, 2248 insertions(+) create mode 100644 go_installer.sh create mode 100644 packages.csv create mode 100644 release-pipeline.yml create mode 100644 setup/esrp_sign_linux.yml create mode 100644 setup/esrp_sign_notarize_macos.yml create mode 100644 setup/esrp_sign_windows.yml create mode 100644 setup/releaseVersionUpdate.py create mode 100644 setup/test_artifacts.yml diff --git a/go_installer.sh b/go_installer.sh new file mode 100644 index 000000000..932951471 --- /dev/null +++ b/go_installer.sh @@ -0,0 +1,14 @@ +#!/bin/bash +work_dir=$(echo $1 | sed 's:/*$::') +version="1.21.8" +arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` + +if [ $arch != "arm64" ];then + arch="amd64" +fi + +echo "Installing on : " $arch " Version : " $version +wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir" +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz +sudo ln -sf /usr/local/go/bin/go /usr/bin/go \ No newline at end of file diff --git a/packages.csv b/packages.csv new file mode 100644 index 000000000..932951471 --- /dev/null +++ b/packages.csv @@ -0,0 +1,14 @@ +#!/bin/bash +work_dir=$(echo $1 | sed 's:/*$::') +version="1.21.8" +arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` + +if [ $arch != "arm64" ];then + arch="amd64" +fi + +echo "Installing on : " $arch " Version : " $version +wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir" +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz +sudo ln -sf /usr/local/go/bin/go /usr/bin/go \ No newline at end of file diff --git a/release-pipeline.yml b/release-pipeline.yml new file mode 100644 index 000000000..581c23549 --- /dev/null +++ b/release-pipeline.yml @@ -0,0 +1,1996 @@ +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' + AZCOPY_VERSION: '0.0.0' + MUTEX_URL: https://ciazcopyv10regular.blob.core.windows.net/distributedmutex/lock + +parameters: + - name: tag + displayName: 'New Release Tag' + type: string + default: 'azcopy-' + + - name: test_mode + displayName: 'Test Mode' + type: boolean + default: false + + # - name: unit_test + # displayName: 'Execute Unit Tests' + # type: boolean + # default: false + + - name: post_release + displayName: 'Post Release on Github' + type: boolean + default: false + + - name: publish_artifacts + displayName: 'Publish Artifacts to Linux Repos' + type: boolean + default: false + + - name: draft + displayName: 'Post as Draft Release' + type: boolean + default: false + + - name: prerelease + displayName: 'Post as PreRelease' + type: boolean + default: false + + - name: update_version + displayName: 'Update Version' + type: boolean + default: false + +# Do not trigger this pipeline automatically +trigger: none +pr: none + +stages: + # - stage: RunTests + # jobs: + # - job: UT_On_Ubuntu + # variables: + # group: AZCOPY_TESTS_VAR + # isMutexSet: 'false' + # # allow maximum build time, in case we have build congestion + # timeoutInMinutes: 360 + # pool: + # vmImage: 'ubuntu-20.04' + # steps: + # - task: UsePythonVersion@0 + # name: 'Set_up_Python' + # inputs: + # versionSpec: '3.7' + + # - task: GoTool@0 + # name: 'Set_up_Golang' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + + # - task: DownloadSecureFile@1 + # name: ciGCSServiceAccountKey + # displayName: 'Download GCS Service Account Key' + # inputs: + # secureFile: 'ci-gcs-dev.json' + + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Install dependencies' + + # - script: | + # pip install azure-storage-blob==12.12.0 + # # set the variable to indicate that the mutex is being acquired + # # note: we set it before acquiring the mutex to ensure we release the mutex. + # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the + # # mutex but before we set this variable. + # # setting this before will always work since it is valid to break an un-acquired mutex. + # echo '##vso[task.setvariable variable=isMutexSet]true' + # # acquire the mutex before running live tests to avoid conflicts + # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" + # name: 'Acquire_the_distributed_mutex' + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'cmd' + # coverage_name: 'cmd' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common' + # coverage_name: 'common' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'common/parallel' + # coverage_name: 'parallel' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'ste' + # coverage_name: 'ste' + + # - template: azurePipelineTemplates/run-ut.yml + # parameters: + # directory: 'sddl' + # coverage_name: 'sddl' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 + # go build -tags "netgo" -o test-validator ./testSuite/ + # mkdir test-temp + # mkdir coverage + # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 + # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator + # export TEST_DIRECTORY_PATH=$(pwd)/test-temp + + # keyctl session test python ./testSuite/scripts/run.py + + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o smoke_coverage.txt + # echo 'Formatting coverage to json format' + # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json + # echo 'Formatting coverage to xml format' + # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml + # name: 'Run_smoke_tests' + # env: + # ACCOUNT_NAME: $(ACCOUNT_NAME) + # ACCOUNT_KEY: $(ACCOUNT_KEY) + # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) + # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) + # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) + # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) + # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) + # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) + # FILESYSTEM_URL: $(FILESYSTEM_URL) + # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) + # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) + # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) + # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) + # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) + # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) + # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) + # SHARE_SAS_URL: $(SHARE_SAS_URL) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # condition: succeededOrFailed() + # # Smoke Tests Publishing + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml + # - script: | + # pip install azure-storage-blob==12.12.0 + # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" + # name: 'Release_the_distributed_mutex' + # # this runs even if the job was canceled (only if the mutex was acquired by this job) + # condition: and(always(), eq(variables['isMutexSet'], 'true')) + # - job: E2E_Test + # timeoutInMinutes: 360 + # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + # strategy: + # matrix: + # Ubuntu-20: + # imageName: 'ubuntu-20.04' + # build_name: 'azcopy_linux_amd64' + # display_name: "Linux" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' + # Windows: + # imageName: 'windows-2019' + # build_name: 'azcopy_windows_amd64.exe' + # display_name: "Windows" + # type: 'windows' + # go_path: 'C:\Users\VssAdministrator\go\bin\' + # suffix: '.exe' + # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' + # output_test_logs: 'cat test.txt' + # save_exit_code: 'set exit_code=%errorlevel%' + # return_exit_code: 'exit %exit_code%' + # go_build: 'go build -cover -o $(build_name)' + # MacOS: + # imageName: 'macos-11' + # build_name: 'azcopy_darwin_amd64' + # display_name: "MacOS" + # go_path: '$(go env GOPATH)/bin/' + # suffix: '' + # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' + # output_test_logs: '' + # save_exit_code: 'exit_code=${PIPESTATUS[0]}' + # return_exit_code: 'exit "$exit_code"' + # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' + # pool: + # vmImage: $(imageName) + # variables: + # - group: AZCOPY_TESTS_VAR + + # steps: + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + # pwsh: 'true' + # displayName: 'Install Powershell Az Module' + # - task: GoTool@0 + # inputs: + # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + # - script: | + # go install github.com/jstemmer/go-junit-report@v0.9.1 + # go install github.com/axw/gocov/gocov@v1.1.0 + # go install github.com/AlekSi/gocov-xml@v1.0.0 + # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + # displayName: 'Installing dependencies' + # - bash: | + # echo "##vso[task.setvariable variable=CGO_ENABLED]0" + # displayName: 'Set CGO_ENABLED for Windows' + # condition: eq(variables.type, 'windows') + # - bash: | + # npm install -g azurite + # mkdir azurite + # azurite --silent --location azurite --debug azurite\debug.log & + # displayName: 'Install and Run Azurite' + # # Running E2E Tests on AMD64 + # - script: | + # mkdir coverage + # echo 'Building executable' + # $(go_build) + # echo 'Running tests' + # $(run_e2e) + # $(save_exit_code) + # echo "Generating junit report" + # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml + # echo 'Formatting coverage directory to legacy txt format' + # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt + # $(output_test_logs) + # echo 'Formatting coverage to json format' + # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json + # echo 'Formatting coverage to xml format' + # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml + # $(return_exit_code) + # env: + # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) + # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) + # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) + # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) + # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) + # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + # displayName: 'E2E Test $(display_name) - AMD64' + + # - task: PublishBuildArtifacts@1 + # displayName: 'Publish logs' + # condition: succeededOrFailed() + # inputs: + # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + # artifactName: logs + + # - task: PublishTestResults@2 + # condition: succeededOrFailed() + # inputs: + # testRunner: JUnit + # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + # testRunTitle: 'Go on $(display_name)' + + # - task: PublishCodeCoverageResults@1 + # condition: succeededOrFailed() + # inputs: + # codeCoverageTool: Cobertura + # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + + - stage: BuildArtifacts + #dependsOn: RunTests + #condition: succeeded('RunTests') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files for packaging' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy_linux_amd64 --version + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + displayName: 'Copy required files' + + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy_linux_arm64 --version + azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # TODO: Add description + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here + + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: windows_amd64 + value: '$(work_dir)/azcopy_windows_amd64' + - name: windows_arm64 + value: '$(work_dir)/azcopy_windows_arm64' + - name: windows_386 + value: '$(work_dir)/azcopy_windows_386' + - name: archives + value: '$(work_dir)/archives' + + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir -p $(windows_amd64) + mkdir -p $(windows_arm64) + mkdir -p $(windows_386) + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: '*.exe' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped + unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped + mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/unzipped + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: '*.exe' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' + + # SignArtifacts end here + + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy_linux_arm64 --version + azcopy_linux_arm64 --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy_linux_arm64 --version + azcopy_linux_arm64 --help + displayName: 'Check Version and Help' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' + + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help + displayName: 'Check Version and Help' + + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # Run the azcopy version command + ./azcopy_darwin_amd64 --version + displayName: 'Install Zip, Extract Files, and Run Version Command' + + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Verify that azcopy.exe exists + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { + Write-Error "azcopy.exe not found in extraction directory: $extractDir" + exit 1 + } + + # Run the azcopy version command + .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version + displayName: 'Extract Files and Run Version Command on Windows' + + # TestArtifacts ends here + + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToContainer + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + - name: Test_Mode + value: ${{ parameters.test_mode }} + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Binaries Build Artifacts' + inputs: + artifactName: 'azCopy-binaries-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get install -y zip + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Create drop.zip file and install azcopy deb package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - task: ArchiveFiles@2 + displayName: 'Archive all binaries to drop.zip' + inputs: + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' + archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' + continueOnError: true + + - task: Bash@3 + displayName: 'Install Azure CLI' + inputs: + targetType: 'inline' + script: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=$(azcopy_linux_amd64 --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + ls -ltR $(Build.ArtifactStagingDirectory) + + today=$(date +"%Y%m%d") + + if [ $(Test_Mode) = "True" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy_linux_amd64 + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - ${{ if eq(parameters.post_release, true) }}: + + # add release tags & push to github + #TODO: update github connection and repo name while merging to main + - task: GithubRelease@1 + inputs: + githubConnection: 'dphulkar-msft-github-rel' + repositoryName: 'dphulkar-msft/azure-storage-azcopy' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + mv azCopy-linux-signed/*tar.gz . + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + displayName: 'Upload files' + workingDirectory: $(Build.ArtifactStagingDirectory) + + # replace org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + # Find the Mariner RPM file and upload + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - stage: UpdateLatestVersion + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - ${{ if eq(parameters.update_version, true) }}: + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' + + # TODO: change org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) diff --git a/setup/esrp_sign_linux.yml b/setup/esrp_sign_linux.yml new file mode 100644 index 000000000..949238cc0 --- /dev/null +++ b/setup/esrp_sign_linux.yml @@ -0,0 +1,25 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_KEY_CODE: '' + Pattern: '' + DisplayName: '' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ${{ parameters.DisplayName }} + inputs: + ConnectedServiceName: 'PMC ESRP AzCopy Signing' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + VerboseLogin: true + inlineOperation: | + [ + { + "KeyCode" : "${{ parameters.ESRP_AZCOPY_KEY_CODE }}", + "OperationCode" : "LinuxSign", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] diff --git a/setup/esrp_sign_notarize_macos.yml b/setup/esrp_sign_notarize_macos.yml new file mode 100644 index 000000000..fb4d4ff1f --- /dev/null +++ b/setup/esrp_sign_notarize_macos.yml @@ -0,0 +1,59 @@ +# esrp-sign-notarize-macos.yml +parameters: + FolderPath: '' + ESRP_AZCOPY_MAC_KEY_CODE: '' + Pattern: '*.zip' + + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppDeveloperSign", + "Parameters":{ + "Hardening":"--options=runtime" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 + +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Notarize Mac executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode":"${{ parameters.ESRP_AZCOPY_MAC_KEY_CODE }}", + "OperationCode":"MacAppNotarize", + "Parameters":{ + "BundleId":"com.microsoft.AzCopy" + }, + "ToolName":"sign", + "ToolVersion":"1.0" + } + ] + SessionTimeout: 30 \ No newline at end of file diff --git a/setup/esrp_sign_windows.yml b/setup/esrp_sign_windows.yml new file mode 100644 index 000000000..1aeb8ee68 --- /dev/null +++ b/setup/esrp_sign_windows.yml @@ -0,0 +1,50 @@ +parameters: + FolderPath: '' + ESRP_AZCOPY_WIN_KEY_CODE: '' + Pattern: '*.exe' + +steps: +- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5 + displayName: 'Sign Windows executable' + inputs: + ConnectedServiceName: 'ESRP KeyVault identity' + AppRegistrationClientId: '77acb45d-e931-4568-9bfc-d0607bdf3759' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'azcopy-esrp-kv' + AuthCertName: 'azcopy-auth' + AuthSignCertName: 'azcopy-sign-test' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "${{ parameters.ESRP_AZCOPY_WIN_KEY_CODE }}", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] + SessionTimeout: 30 diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py new file mode 100644 index 000000000..b70c0474e --- /dev/null +++ b/setup/releaseVersionUpdate.py @@ -0,0 +1,50 @@ +import json +import requests +import sys +import os +from xml.dom import minidom +from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient + + +sasUrl = sys.argv[1] +releaseVersion = sys.argv[2].split(' ')[2] +print('Release Version: ' + releaseVersion) +if(len(releaseVersion)==0): + print('Incorrect Release Version') + sys.exit(1) + +containerUrl = sasUrl.split('?')[0] +sasToken = sasUrl.split('?')[1] + +# delete latest version file in the container +# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +# print(f"deleteURL: '{deleteUrl}'") +# resp = requests.delete(deleteUrl) +# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') + +# Create a file and write data to it +file_name = 'latest_version.txt' + +# Open the file in write mode +with open(file_name, 'w') as file: + file.write(releaseVersion) +print(f'Data written to {file_name}') + +# Create the BlobServiceClient using the SAS URL +blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) + +# Get the container name from the SAS URL +container_name = sasUrl.split('?')[0].split('/')[-1] + +# Get a client to interact with the container +container_client = blob_service_client.get_container_client(container_name) + +# Upload the file to the container +with open(file_name, 'rb') as data: + container_client.upload_blob(name=file_name, data=data, overwrite=True) + +print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + +# Clean up the local file +os.remove(file_name) +print(f"Local file '{file_name}' deleted.") diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml new file mode 100644 index 000000000..f09b47044 --- /dev/null +++ b/setup/test_artifacts.yml @@ -0,0 +1,40 @@ +parameters: + - name: artifact_name + type: string + - name: download_path + type: string + - name: item_pattern + type: string + +steps: + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + + - task: ShellScript@2 + inputs: + scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" + args: "$(System.DefaultWorkingDirectory)/" + displayName: "GoTool Custom Setup" + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: ${{ parameters.artifact_name }} + downloadPath: ${{ parameters.download_path }} + itemPattern: ${{ parameters.item_pattern }} + + - script: | + ls -l + result=$(ls -1 | wc -l) + if [ $result -ne 1 ]; then + exit 1 + fi + displayName: 'List Downloaded Package' + workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} From 1574146f5be0dc76a2241265a055741630a1771a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 20 Jun 2024 11:25:51 +0530 Subject: [PATCH 301/513] Release pipeline for linux, mariner, mac and windows --- release-pipelines.yml | 27 ++++++++++++++------------- setup/releaseVersionUpdate.py | 8 +++----- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a908588b6..6f7f95b0d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1187,11 +1187,12 @@ stages: displayName: 'Publish Signed Artifacts' - script: | - rm -rf $(Build.ArtifactStagingDirectory)/*.zip + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped - unzip -o $(Build.ArtifactStagingDirectory)/*.zip -d $(Build.ArtifactStagingDirectory)/unzipped + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped mv $(Build.ArtifactStagingDirectory)/unzipped/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ rm -rf $(Build.ArtifactStagingDirectory)/unzipped + rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Copy signed mac binary to staging directory' @@ -1316,8 +1317,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help displayName: 'Check Version and Help' - job: Set_2 @@ -1354,8 +1355,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_arm64 --version + azcopy_linux_arm64 --help displayName: 'Check Version and Help' - job: Set_3 @@ -1394,8 +1395,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_arm64 --version + azcopy_linux_arm64 --help displayName: 'Check Version and Help' - job: Set_4 @@ -1439,8 +1440,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy --version - azcopy --help + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help displayName: 'Check Version and Help' - job: Set_5 @@ -1483,8 +1484,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | - azcopy --version - azcopy --help + azcopy_linux_amd64 --version + azcopy_linux_amd64 --help displayName: 'Check Version and Help' - job: Set_6 @@ -1725,7 +1726,7 @@ stages: # Upload the archived builds AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index b70c0474e..95e1d4b27 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -17,14 +17,12 @@ sasToken = sasUrl.split('?')[1] # delete latest version file in the container -# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken -# print(f"deleteURL: '{deleteUrl}'") -# resp = requests.delete(deleteUrl) -# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') +deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +resp = requests.delete(deleteUrl) +sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') # Create a file and write data to it file_name = 'latest_version.txt' - # Open the file in write mode with open(file_name, 'w') as file: file.write(releaseVersion) From 0b8dd9c6e4d00b7c5819659765082507d73d6f95 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 24 Jun 2024 11:25:29 +0530 Subject: [PATCH 302/513] test azcopy --- release-pipelines.yml | 343 ++++++------------------------------------ 1 file changed, 43 insertions(+), 300 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6ba491fff..198c774eb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -14,11 +14,6 @@ parameters: displayName: 'Test Mode' type: boolean default: false - - # - name: unit_test - # displayName: 'Execute Unit Tests' - # type: boolean - # default: false - name: post_release displayName: 'Post Release on Github' @@ -50,268 +45,7 @@ trigger: none pr: none stages: - # - stage: RunTests - # jobs: - # - job: UT_On_Ubuntu - # variables: - # group: AZCOPY_TESTS_VAR - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py - - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) - # variables: - # - group: AZCOPY_TESTS_VAR - - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - - stage: BuildArtifacts - #dependsOn: RunTests - #condition: succeeded('RunTests') jobs: - job: Set_1 timeoutInMinutes: 120 @@ -409,7 +143,9 @@ stages: - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy + mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy + cp $(linux_amd64)/azcopy pkgDir/usr/bin/ cp NOTICE.txt $(linux_amd64)/ cp NOTICE.txt $(linux_se_amd64)/ displayName: 'Copy required files for packaging' @@ -421,11 +157,11 @@ stages: - script: | export GOARCH=amd64 export GOOS=linux - $(linux_amd64)/azcopy_linux_amd64 --version - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + $(linux_amd64)/azcopy --version + azcopyOutput=$($(linux_amd64)/azcopy --version) # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + azcopyOutput=$($(linux_amd64)/azcopy --version) # Extract the version part using grep and sed if echo "$azcopyOutput" | grep -q "azcopy version"; then @@ -449,22 +185,22 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -472,7 +208,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Partner(SE) Linux Build' inputs: - rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + rootFolderOrFile: '$(linux_se_amd64)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' continueOnError: true @@ -480,7 +216,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Amd Linux Build' inputs: - rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + rootFolderOrFile: '$(linux_amd64)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' continueOnError: true @@ -599,15 +335,17 @@ stages: - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ cp NOTICE.txt $(linux_se_arm64)/ + cp NOTICE.txt $(linux_arm64)/ + mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy + cp $(linux_arm64)/azcopy pkgDir/usr/bin/ displayName: 'Copy required files' - script: | export GOARCH=arm64 export GOOS=linux - $(linux_arm64)/azcopy_linux_arm64 --version - azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + $(linux_arm64)/azcopy --version + azcopyOutput=$($(linux_arm64)/azcopy --version) # Extract the version part using grep and sed if echo "$azcopyOutput" | grep -q "azcopy version"; then @@ -632,22 +370,22 @@ stages: # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' @@ -663,7 +401,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Partner(SE) Linux Build' inputs: - rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + rootFolderOrFile: '$(linux_se_arm64)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' continueOnError: true @@ -842,6 +580,8 @@ stages: - script: | cp NOTICE.txt $(darwin_amd_dir) cp NOTICE.txt $(darwin_arm_dir) + mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy displayName: 'Copy NOTICE file' - task: ArchiveFiles@2 @@ -1071,6 +811,9 @@ stages: cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) + mv $(windows_amd64)/azcopy_windows_amd64.exe $(windows_amd64)/azcopy + mv $(windows_arm64)/azcopy_windows_arm64.exe $(windows_arm64)/azcopy + mv $(windows_386)/azcopy_windows_386.exe $(windows_386)/azcopy zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . @@ -1191,8 +934,8 @@ stages: mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) @@ -1319,8 +1062,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_amd64 --version - azcopy_linux_amd64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_2 @@ -1357,8 +1100,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_arm64 --version - azcopy_linux_arm64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_3 @@ -1397,8 +1140,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_arm64 --version - azcopy_linux_arm64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_4 @@ -1442,8 +1185,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_amd64 --version - azcopy_linux_amd64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_5 @@ -1486,8 +1229,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | - azcopy_linux_amd64 --version - azcopy_linux_amd64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_6 @@ -1549,7 +1292,7 @@ stages: cd $EXTRACT_DIR/azcopy_darwin_amd64 # Run the azcopy version command - ./azcopy_darwin_amd64 --version + ./azcopy --version displayName: 'Install Zip, Extract Files, and Run Version Command' - job: Set_7 @@ -1625,7 +1368,7 @@ stages: } # Run the azcopy version command - .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version + .\azcopy_windows_amd64\azcopy.exe --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1710,7 +1453,7 @@ stages: scriptLocation: inlineScript inlineScript: | # Find the current version - raw_version_output=$(azcopy_linux_amd64 --version) + raw_version_output=$(azcopy --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') ls -ltR $(Build.ArtifactStagingDirectory) @@ -1724,7 +1467,7 @@ stages: fi echo "Artifacts will be uploaded to: $container_url" - executable_name=azcopy_linux_amd64 + executable_name=azcopy # Upload the archived builds AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true From 569d36fa0d8070e92f6a0b106e0f7927a7951f4b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 24 Jun 2024 11:25:29 +0530 Subject: [PATCH 303/513] test azcopy --- release-pipelines.yml | 343 +++++----------------------------- setup/releaseVersionUpdate.py | 2 +- 2 files changed, 44 insertions(+), 301 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6ba491fff..983b1b5f3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -14,11 +14,6 @@ parameters: displayName: 'Test Mode' type: boolean default: false - - # - name: unit_test - # displayName: 'Execute Unit Tests' - # type: boolean - # default: false - name: post_release displayName: 'Post Release on Github' @@ -50,268 +45,7 @@ trigger: none pr: none stages: - # - stage: RunTests - # jobs: - # - job: UT_On_Ubuntu - # variables: - # group: AZCOPY_TESTS_VAR - # isMutexSet: 'false' - # # allow maximum build time, in case we have build congestion - # timeoutInMinutes: 360 - # pool: - # vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # name: 'Set_up_Python' - # inputs: - # versionSpec: '3.7' - - # - task: GoTool@0 - # name: 'Set_up_Golang' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - - # - task: DownloadSecureFile@1 - # name: ciGCSServiceAccountKey - # displayName: 'Download GCS Service Account Key' - # inputs: - # secureFile: 'ci-gcs-dev.json' - - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Install dependencies' - - # - script: | - # pip install azure-storage-blob==12.12.0 - # # set the variable to indicate that the mutex is being acquired - # # note: we set it before acquiring the mutex to ensure we release the mutex. - # # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the - # # mutex but before we set this variable. - # # setting this before will always work since it is valid to break an un-acquired mutex. - # echo '##vso[task.setvariable variable=isMutexSet]true' - # # acquire the mutex before running live tests to avoid conflicts - # python ./tool_distributed_mutex.py lock "$(MUTEX_URL)" - # name: 'Acquire_the_distributed_mutex' - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'cmd' - # coverage_name: 'cmd' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common' - # coverage_name: 'common' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'common/parallel' - # coverage_name: 'parallel' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'ste' - # coverage_name: 'ste' - - # - template: azurePipelineTemplates/run-ut.yml - # parameters: - # directory: 'sddl' - # coverage_name: 'sddl' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 - # go build -tags "netgo" -o test-validator ./testSuite/ - # mkdir test-temp - # mkdir coverage - # export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64 - # export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator - # export TEST_DIRECTORY_PATH=$(pwd)/test-temp - - # keyctl session test python ./testSuite/scripts/run.py - - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o smoke_coverage.txt - # echo 'Formatting coverage to json format' - # $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json - # echo 'Formatting coverage to xml format' - # $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml - # name: 'Run_smoke_tests' - # env: - # ACCOUNT_NAME: $(ACCOUNT_NAME) - # ACCOUNT_KEY: $(ACCOUNT_KEY) - # AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - # AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - # GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT) - # GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath) - # ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL) - # CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL) - # CONTAINER_SAS_URL: $(CONTAINER_SAS_URL) - # FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL) - # FILESYSTEM_URL: $(FILESYSTEM_URL) - # OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT) - # OAUTH_TENANT_ID: $(OAUTH_TENANT_ID) - # PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL) - # S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL) - # S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL) - # S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL) - # S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL) - # SHARE_SAS_URL: $(SHARE_SAS_URL) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # condition: succeededOrFailed() - # # Smoke Tests Publishing - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml - # - script: | - # pip install azure-storage-blob==12.12.0 - # python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)" - # name: 'Release_the_distributed_mutex' - # # this runs even if the job was canceled (only if the mutex was acquired by this job) - # condition: and(always(), eq(variables['isMutexSet'], 'true')) - # - job: E2E_Test - # timeoutInMinutes: 360 - # # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 - # strategy: - # matrix: - # Ubuntu-20: - # imageName: 'ubuntu-20.04' - # build_name: 'azcopy_linux_amd64' - # display_name: "Linux" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'GOARCH=amd64 GOOS=linux go build -tags "netgo" -o $(build_name)' - # Windows: - # imageName: 'windows-2019' - # build_name: 'azcopy_windows_amd64.exe' - # display_name: "Windows" - # type: 'windows' - # go_path: 'C:\Users\VssAdministrator\go\bin\' - # suffix: '.exe' - # run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt' - # output_test_logs: 'cat test.txt' - # save_exit_code: 'set exit_code=%errorlevel%' - # return_exit_code: 'exit %exit_code%' - # go_build: 'go build -cover -o $(build_name)' - # MacOS: - # imageName: 'macos-11' - # build_name: 'azcopy_darwin_amd64' - # display_name: "MacOS" - # go_path: '$(go env GOPATH)/bin/' - # suffix: '' - # run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt' - # output_test_logs: '' - # save_exit_code: 'exit_code=${PIPESTATUS[0]}' - # return_exit_code: 'exit "$exit_code"' - # go_build: 'CGO_ENABLED=1 go build -cover -o $(build_name)' - # pool: - # vmImage: $(imageName) - # variables: - # - group: AZCOPY_TESTS_VAR - - # steps: - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' - # pwsh: 'true' - # displayName: 'Install Powershell Az Module' - # - task: GoTool@0 - # inputs: - # version: $(AZCOPY_GOLANG_VERSION_COVERAGE) - # - script: | - # go install github.com/jstemmer/go-junit-report@v0.9.1 - # go install github.com/axw/gocov/gocov@v1.1.0 - # go install github.com/AlekSi/gocov-xml@v1.0.0 - # go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b - # displayName: 'Installing dependencies' - # - bash: | - # echo "##vso[task.setvariable variable=CGO_ENABLED]0" - # displayName: 'Set CGO_ENABLED for Windows' - # condition: eq(variables.type, 'windows') - # - bash: | - # npm install -g azurite - # mkdir azurite - # azurite --silent --location azurite --debug azurite\debug.log & - # displayName: 'Install and Run Azurite' - # # Running E2E Tests on AMD64 - # - script: | - # mkdir coverage - # echo 'Building executable' - # $(go_build) - # echo 'Running tests' - # $(run_e2e) - # $(save_exit_code) - # echo "Generating junit report" - # cat test.txt | $(go_path)go-junit-report$(suffix) > $(display_name)_report.xml - # echo 'Formatting coverage directory to legacy txt format' - # go tool covdata textfmt -i=coverage -o $(display_name)_coverage.txt - # $(output_test_logs) - # echo 'Formatting coverage to json format' - # $(go_path)gocov$(suffix) convert $(display_name)_coverage.txt > $(display_name)_coverage.json - # echo 'Formatting coverage to xml format' - # $(go_path)gocov-xml$(suffix) < $(display_name)_coverage.json > $(display_name)_coverage.xml - # $(return_exit_code) - # env: - # AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) - # AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) - # AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) - # AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) - # AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID) - # AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET) - # AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) - # AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) - # AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' - # AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) - # AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) - # CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) - # CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) - # AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' - # NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) - # NEW_E2E_APPLICATION_ID: $(AZCOPY_NEW_E2E_APPLICATION_ID) - # NEW_E2E_CLIENT_SECRET: $(AZCOPY_NEW_E2E_CLIENT_SECRET) - # NEW_E2E_TENANT_ID: $(OAUTH_TENANT_ID) - # NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) - # displayName: 'E2E Test $(display_name) - AMD64' - - # - task: PublishBuildArtifacts@1 - # displayName: 'Publish logs' - # condition: succeededOrFailed() - # inputs: - # pathToPublish: '$(System.DefaultWorkingDirectory)/logs' - # artifactName: logs - - # - task: PublishTestResults@2 - # condition: succeededOrFailed() - # inputs: - # testRunner: JUnit - # testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml - # testRunTitle: 'Go on $(display_name)' - - # - task: PublishCodeCoverageResults@1 - # condition: succeededOrFailed() - # inputs: - # codeCoverageTool: Cobertura - # summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml - # additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html - - stage: BuildArtifacts - #dependsOn: RunTests - #condition: succeeded('RunTests') jobs: - job: Set_1 timeoutInMinutes: 120 @@ -409,7 +143,9 @@ stages: - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_amd64)/azcopy_linux_amd64 pkgDir/usr/bin/ + mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy + mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy + cp $(linux_amd64)/azcopy pkgDir/usr/bin/ cp NOTICE.txt $(linux_amd64)/ cp NOTICE.txt $(linux_se_amd64)/ displayName: 'Copy required files for packaging' @@ -421,11 +157,11 @@ stages: - script: | export GOARCH=amd64 export GOOS=linux - $(linux_amd64)/azcopy_linux_amd64 --version - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + $(linux_amd64)/azcopy --version + azcopyOutput=$($(linux_amd64)/azcopy --version) # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy_linux_amd64 --version) + azcopyOutput=$($(linux_amd64)/azcopy --version) # Extract the version part using grep and sed if echo "$azcopyOutput" | grep -q "azcopy version"; then @@ -449,22 +185,22 @@ stages: # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_amd64 --version | cut -d " " -f 3`.x86_64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' @@ -472,7 +208,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Partner(SE) Linux Build' inputs: - rootFolderOrFile: '$(linux_se_amd64)/azcopy_linux_se_amd64' + rootFolderOrFile: '$(linux_se_amd64)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' continueOnError: true @@ -480,7 +216,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Amd Linux Build' inputs: - rootFolderOrFile: '$(linux_amd64)/azcopy_linux_amd64' + rootFolderOrFile: '$(linux_amd64)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' continueOnError: true @@ -599,15 +335,17 @@ stages: - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - cp $(linux_arm64)/azcopy_linux_arm64 pkgDir/usr/bin/ cp NOTICE.txt $(linux_se_arm64)/ + cp NOTICE.txt $(linux_arm64)/ + mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy + cp $(linux_arm64)/azcopy pkgDir/usr/bin/ displayName: 'Copy required files' - script: | export GOARCH=arm64 export GOOS=linux - $(linux_arm64)/azcopy_linux_arm64 --version - azcopyOutput=$($(linux_arm64)/azcopy_linux_arm64 --version) + $(linux_arm64)/azcopy --version + azcopyOutput=$($(linux_arm64)/azcopy --version) # Extract the version part using grep and sed if echo "$azcopyOutput" | grep -q "azcopy version"; then @@ -632,22 +370,22 @@ stages: # TODO: Add description - script: | fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.deb + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 deb Package' - script: | fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3` \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy_linux_arm64 --version | cut -d " " -f 3`.arm64.rpm + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' @@ -663,7 +401,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Partner(SE) Linux Build' inputs: - rootFolderOrFile: '$(linux_se_arm64)/azcopy_linux_se_arm64' + rootFolderOrFile: '$(linux_se_arm64)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' continueOnError: true @@ -842,6 +580,8 @@ stages: - script: | cp NOTICE.txt $(darwin_amd_dir) cp NOTICE.txt $(darwin_arm_dir) + mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy displayName: 'Copy NOTICE file' - task: ArchiveFiles@2 @@ -1071,6 +811,9 @@ stages: cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) + mv $(windows_amd64)/azcopy_windows_amd64.exe $(windows_amd64)/azcopy + mv $(windows_arm64)/azcopy_windows_arm64.exe $(windows_arm64)/azcopy + mv $(windows_386)/azcopy_windows_386.exe $(windows_386)/azcopy zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . @@ -1191,8 +934,8 @@ stages: mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy_darwin* $(Build.ArtifactStagingDirectory)/ + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) @@ -1319,8 +1062,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_amd64 --version - azcopy_linux_amd64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_2 @@ -1357,8 +1100,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_arm64 --version - azcopy_linux_arm64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_3 @@ -1397,8 +1140,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_arm64 --version - azcopy_linux_arm64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_4 @@ -1442,8 +1185,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - azcopy_linux_amd64 --version - azcopy_linux_amd64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_5 @@ -1486,8 +1229,8 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | - azcopy_linux_amd64 --version - azcopy_linux_amd64 --help + azcopy --version + azcopy --help displayName: 'Check Version and Help' - job: Set_6 @@ -1549,7 +1292,7 @@ stages: cd $EXTRACT_DIR/azcopy_darwin_amd64 # Run the azcopy version command - ./azcopy_darwin_amd64 --version + ./azcopy --version displayName: 'Install Zip, Extract Files, and Run Version Command' - job: Set_7 @@ -1625,7 +1368,7 @@ stages: } # Run the azcopy version command - .\azcopy_windows_amd64\azcopy_windows_amd64.exe --version + .\azcopy_windows_amd64\azcopy --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1710,7 +1453,7 @@ stages: scriptLocation: inlineScript inlineScript: | # Find the current version - raw_version_output=$(azcopy_linux_amd64 --version) + raw_version_output=$(azcopy --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') ls -ltR $(Build.ArtifactStagingDirectory) @@ -1724,7 +1467,7 @@ stages: fi echo "Artifacts will be uploaded to: $container_url" - executable_name=azcopy_linux_amd64 + executable_name=azcopy # Upload the archived builds AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index 4e577aa42..95e1d4b27 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -17,7 +17,7 @@ sasToken = sasUrl.split('?')[1] # delete latest version file in the container -deleteUrl = containerUrl + '/' + 'releasemetadata/latest_version.txt' + '?' + sasToken +deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken resp = requests.delete(deleteUrl) sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') From 8b9a694036e3f482e44eb66e2c3fff894c04a66d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 24 Jun 2024 14:54:34 +0530 Subject: [PATCH 304/513] test azcopy --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 983b1b5f3..98133414a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1362,7 +1362,7 @@ stages: Set-Location -Path $extractDir # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy_windows_amd64.exe")) { + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy.exe")) { Write-Error "azcopy.exe not found in extraction directory: $extractDir" exit 1 } From 16ba0d049e529c6005eab937761121b9654a7f2c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 24 Jun 2024 15:46:05 +0530 Subject: [PATCH 305/513] test azcopy --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 98133414a..c9e718a55 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1362,7 +1362,7 @@ stages: Set-Location -Path $extractDir # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy.exe")) { + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { Write-Error "azcopy.exe not found in extraction directory: $extractDir" exit 1 } From 8243bb25b4287078fabf84d3798154b479f12636 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 25 Jun 2024 11:10:15 +0530 Subject: [PATCH 306/513] test windows --- release-pipelines.yml | 1952 ++++++++++++++++++++--------------------- 1 file changed, 976 insertions(+), 976 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c9e718a55..52fbe7e05 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -47,381 +47,381 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_amd64 - value: $(work_dir)/linux_amd64 - - name: linux_se_amd64 - value: $(work_dir)/linux_se_amd64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_amd64) - mkdir $(linux_se_amd64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(linux_amd64)/ - sudo ls -lRt $(linux_se_amd64)/ - cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - workingDirectory: $(root_dir) - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_amd64 + # value: $(work_dir)/linux_amd64 + # - name: linux_se_amd64 + # value: $(work_dir)/linux_se_amd64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_amd64) + # mkdir $(linux_se_amd64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_amd64)/ + # sudo ls -lRt $(linux_se_amd64)/ + # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + # workingDirectory: $(root_dir) + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy - mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy - cp $(linux_amd64)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64)/ - cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files for packaging' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=amd64 - export GOOS=linux - $(linux_amd64)/azcopy --version - azcopyOutput=$($(linux_amd64)/azcopy --version) - - # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy + # mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy + # cp $(linux_amd64)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64)/ + # cp NOTICE.txt $(linux_se_amd64)/ + # displayName: 'Copy required files for packaging' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # $(linux_amd64)/azcopy --version + # azcopyOutput=$($(linux_amd64)/azcopy --version) + + # # Run azcopy --version and capture the output + # azcopyOutput=$($(linux_amd64)/azcopy --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_arm64 - value: $(work_dir)/linux_arm64 - - name: linux_se_arm64 - value: $(work_dir)/linux_se_arm64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_arm64) - mkdir $(linux_se_arm64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_arm64 + # value: $(work_dir)/linux_arm64 + # - name: linux_se_arm64 + # value: $(work_dir)/linux_se_arm64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_arm64) + # mkdir $(linux_se_arm64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(linux_arm64)/ - sudo ls -lRt $(linux_se_arm64)/ - cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - workingDirectory: $(root_dir) - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_arm64)/ + # sudo ls -lRt $(linux_se_arm64)/ + # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + # workingDirectory: $(root_dir) + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp NOTICE.txt $(linux_se_arm64)/ - cp NOTICE.txt $(linux_arm64)/ - mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy - cp $(linux_arm64)/azcopy pkgDir/usr/bin/ - displayName: 'Copy required files' - - - script: | - export GOARCH=arm64 - export GOOS=linux - $(linux_arm64)/azcopy --version - azcopyOutput=$($(linux_arm64)/azcopy --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_se_arm64)/ + # cp NOTICE.txt $(linux_arm64)/ + # mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy + # cp $(linux_arm64)/azcopy pkgDir/usr/bin/ + # displayName: 'Copy required files' + + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # $(linux_arm64)/azcopy --version + # azcopyOutput=$($(linux_arm64)/azcopy --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # TODO: Add description - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # # TODO: Add description + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -505,210 +505,210 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: darwin_amd_dir - value: '$(work_dir)/azcopy_darwin_amd64' - - name: darwin_arm_dir - value: '$(work_dir)/azcopy_darwin_arm64' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - mkdir $(darwin_amd_dir) - mkdir $(darwin_arm_dir) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: darwin_amd_dir + # value: '$(work_dir)/azcopy_darwin_amd64' + # - name: darwin_arm_dir + # value: '$(work_dir)/azcopy_darwin_arm64' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # mkdir $(darwin_amd_dir) + # mkdir $(darwin_arm_dir) + # mkdir $(archives) + # displayName: 'Create required directories' + + # - script: | + # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - script: | - cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - displayName: "Copy binaries to staging directory" - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - displayName: "Remove binaries from staging directory" - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy NOTICE file' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + # displayName: "Copy binaries to staging directory" + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # displayName: "Remove binaries from staging directory" + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy NOTICE file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, *tar.gz' - DisplayName: 'ESRP CodeSigning azCopy linux' + jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/mariner - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - Pattern: '*.rpm, *.deb' - DisplayName: 'ESRP CodeSigning azCopy mariner' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: '*.rpm, *.deb, *tar.gz' + # DisplayName: 'ESRP CodeSigning azCopy linux' + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/mariner + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + # Pattern: '*.rpm, *.deb' + # DisplayName: 'ESRP CodeSigning azCopy mariner' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -811,9 +811,9 @@ stages: cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) - mv $(windows_amd64)/azcopy_windows_amd64.exe $(windows_amd64)/azcopy - mv $(windows_arm64)/azcopy_windows_arm64.exe $(windows_arm64)/azcopy - mv $(windows_386)/azcopy_windows_386.exe $(windows_386)/azcopy + mv $(windows_amd64)/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(windows_arm64)/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(windows_386)/azcopy_windows_386.exe $(windows_386)/azcopy.exe zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . @@ -858,171 +858,171 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac binary to staging directory' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy signed mac binary to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: '*.exe' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: '*.exe' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1030,270 +1030,270 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(System.DefaultWorkingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi - - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # Change to the directory containing azcopy - cd $EXTRACT_DIR/azcopy_darwin_amd64 - - # Run the azcopy version command - ./azcopy --version - displayName: 'Install Zip, Extract Files, and Run Version Command' + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo rpm -qip azcopy*x86_64.rpm + # sudo rpm -i azcopy*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi + + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # # Create extraction directory + # mkdir -p $EXTRACT_DIR + + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # # Change to the directory containing azcopy + # cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # # Run the azcopy version command + # ./azcopy --version + # displayName: 'Install Zip, Extract Files, and Run Version Command' - job: Set_7 timeoutInMinutes: 120 From e066f8b66c20630786b7641fac0336163e740c26 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 25 Jun 2024 11:29:57 +0530 Subject: [PATCH 307/513] test windows --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 52fbe7e05..0dc02512f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1362,13 +1362,13 @@ stages: Set-Location -Path $extractDir # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy.exe")) { Write-Error "azcopy.exe not found in extraction directory: $extractDir" exit 1 } # Run the azcopy version command - .\azcopy_windows_amd64\azcopy --version + .\azcopy_windows_amd64\azcopy.exe --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here From 79ee738cb2055e47327da63eec97e103c74d7be3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 25 Jun 2024 12:52:01 +0530 Subject: [PATCH 308/513] test --- release-pipelines.yml | 1946 ++++++++++++++++++++--------------------- 1 file changed, 970 insertions(+), 976 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0dc02512f..ce998c186 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -47,381 +47,380 @@ pr: none stages: - stage: BuildArtifacts jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_amd64 - # value: $(work_dir)/linux_amd64 - # - name: linux_se_amd64 - # value: $(work_dir)/linux_se_amd64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_amd64) - # mkdir $(linux_se_amd64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_amd64)/ - # sudo ls -lRt $(linux_se_amd64)/ - # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - # workingDirectory: $(root_dir) - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy - # mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy - # cp $(linux_amd64)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64)/ - # cp NOTICE.txt $(linux_se_amd64)/ - # displayName: 'Copy required files for packaging' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # $(linux_amd64)/azcopy --version - # azcopyOutput=$($(linux_amd64)/azcopy --version) - - # # Run azcopy --version and capture the output - # azcopyOutput=$($(linux_amd64)/azcopy --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy + mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy + cp $(linux_amd64)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files for packaging' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy --version + azcopyOutput=$($(linux_amd64)/azcopy --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_arm64 - # value: $(work_dir)/linux_arm64 - # - name: linux_se_arm64 - # value: $(work_dir)/linux_se_arm64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_arm64) - # mkdir $(linux_se_arm64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_arm64)/ - # sudo ls -lRt $(linux_se_arm64)/ - # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - # workingDirectory: $(root_dir) - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_se_arm64)/ - # cp NOTICE.txt $(linux_arm64)/ - # mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy - # cp $(linux_arm64)/azcopy pkgDir/usr/bin/ - # displayName: 'Copy required files' - - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # $(linux_arm64)/azcopy --version - # azcopyOutput=$($(linux_arm64)/azcopy --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + cp NOTICE.txt $(linux_arm64)/ + mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy + cp $(linux_arm64)/azcopy pkgDir/usr/bin/ + displayName: 'Copy required files' + + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy --version + azcopyOutput=$($(linux_arm64)/azcopy --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -505,210 +504,209 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: darwin_amd_dir - # value: '$(work_dir)/azcopy_darwin_amd64' - # - name: darwin_arm_dir - # value: '$(work_dir)/azcopy_darwin_arm64' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # mkdir $(darwin_amd_dir) - # mkdir $(darwin_arm_dir) - # mkdir $(archives) - # displayName: 'Create required directories' - - # - script: | - # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' - # - script: | - # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - # displayName: "Copy binaries to staging directory" - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # displayName: "Remove binaries from staging directory" - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy NOTICE file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: '*.rpm, *.deb, *tar.gz' - # DisplayName: 'ESRP CodeSigning azCopy linux' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/mariner - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - # Pattern: '*.rpm, *.deb' - # DisplayName: 'ESRP CodeSigning azCopy mariner' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -740,7 +738,6 @@ stages: echo ${{ parameters.tag }} displayName: 'Tag Name' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: @@ -811,9 +808,9 @@ stages: cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) - mv $(windows_amd64)/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(windows_arm64)/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(windows_386)/azcopy_windows_386.exe $(windows_386)/azcopy.exe + mv $(windows_amd64)/azcopy_windows_amd64 $(windows_amd64)/azcopy + mv $(windows_arm64)/azcopy_windows_arm64 $(windows_arm64)/azcopy + mv $(windows_386)/azcopy_windows_386 $(windows_386)/azcopy zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . @@ -858,171 +855,170 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy signed mac binary to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: '*.exe' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: '*.exe' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1030,270 +1026,270 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Change to the directory containing azcopy - # cd $EXTRACT_DIR/azcopy_darwin_amd64 - - # # Run the azcopy version command - # ./azcopy --version - # displayName: 'Install Zip, Extract Files, and Run Version Command' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # Run the azcopy version command + ./azcopy --version + displayName: 'Install Zip, Extract Files, and Run Version Command' - job: Set_7 timeoutInMinutes: 120 @@ -1362,13 +1358,13 @@ stages: Set-Location -Path $extractDir # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy.exe")) { + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { Write-Error "azcopy.exe not found in extraction directory: $extractDir" exit 1 } # Run the azcopy version command - .\azcopy_windows_amd64\azcopy.exe --version + .\azcopy_windows_amd64\azcopy --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1494,7 +1490,6 @@ stages: echo ${{ parameters.tag }} displayName: 'Tag Name' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Linux Build Artifacts' inputs: @@ -1712,7 +1707,6 @@ stages: python3 --version displayName: 'Installing Python' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: From 19e9cf24f361fb05ad98843f67ae8378aeeb7bfd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 25 Jun 2024 12:52:01 +0530 Subject: [PATCH 309/513] test --- packages.csv | 2 +- release-pipelines.yml | 1972 ++++++++++++++++++++--------------------- 2 files changed, 983 insertions(+), 991 deletions(-) diff --git a/packages.csv b/packages.csv index 8d03e6d48..120605c75 100644 --- a/packages.csv +++ b/packages.csv @@ -8,7 +8,7 @@ Ubuntu-24.04,azcopyAmdDeb,microsoft-ubuntu-noble-prod-apt,noble Ubuntu-24.04,azcopyArmDeb,microsoft-ubuntu-noble-prod-apt,noble Debian-9.0,azcopyAmdDeb,microsoft-debian-stretch-prod-apt,stretch Debian-10.0,azcopyAmdDeb,microsoft-debian-buster-prod-apt,buster -Debian-11.0,fuse3AmdDeb,microsoft-debian-bullseye-prod-apt,bullseye +Debian-11.0,azcopyAmdDeb,microsoft-debian-bullseye-prod-apt,bullseye RHEL-7.5,azcopyAmdRpm,microsoft-rhel7.5-prod-yum, RHEL-7.8,azcopyAmdRpm,microsoft-rhel7.8-prod-yum, RHEL-8.1,azcopyAmdRpm,microsoft-rhel8.1-prod-yum, diff --git a/release-pipelines.yml b/release-pipelines.yml index 0dc02512f..b70bb497f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1,8 +1,5 @@ variables: AZCOPY_GOLANG_VERSION: '1.21.8' - AZCOPY_GOLANG_VERSION_COVERAGE: '1.21.8' - AZCOPY_VERSION: '0.0.0' - MUTEX_URL: https://ciazcopyv10regular.blob.core.windows.net/distributedmutex/lock parameters: - name: tag @@ -47,381 +44,380 @@ pr: none stages: - stage: BuildArtifacts jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_amd64 - # value: $(work_dir)/linux_amd64 - # - name: linux_se_amd64 - # value: $(work_dir)/linux_se_amd64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_amd64) - # mkdir $(linux_se_amd64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_amd64)/ - # sudo ls -lRt $(linux_se_amd64)/ - # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - # workingDirectory: $(root_dir) - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_amd64 + value: $(work_dir)/linux_amd64 + - name: linux_se_amd64 + value: $(work_dir)/linux_se_amd64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_amd64) + mkdir $(linux_se_amd64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(linux_amd64)/ + sudo ls -lRt $(linux_se_amd64)/ + cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy - # mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy - # cp $(linux_amd64)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64)/ - # cp NOTICE.txt $(linux_se_amd64)/ - # displayName: 'Copy required files for packaging' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # $(linux_amd64)/azcopy --version - # azcopyOutput=$($(linux_amd64)/azcopy --version) - - # # Run azcopy --version and capture the output - # azcopyOutput=$($(linux_amd64)/azcopy --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy + mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy + cp $(linux_amd64)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64)/ + cp NOTICE.txt $(linux_se_amd64)/ + displayName: 'Copy required files for packaging' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + - script: | + export GOARCH=amd64 + export GOOS=linux + $(linux_amd64)/azcopy --version + azcopyOutput=$($(linux_amd64)/azcopy --version) + + # Run azcopy --version and capture the output + azcopyOutput=$($(linux_amd64)/azcopy --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_arm64 - # value: $(work_dir)/linux_arm64 - # - name: linux_se_arm64 - # value: $(work_dir)/linux_se_arm64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_arm64) - # mkdir $(linux_se_arm64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + - name: linux_arm64 + value: $(work_dir)/linux_arm64 + - name: linux_se_arm64 + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + mkdir $(linux_arm64) + mkdir $(linux_se_arm64) + mkdir $(archives) + displayName: "Create required directories" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_arm64)/ - # sudo ls -lRt $(linux_se_arm64)/ - # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - # workingDirectory: $(root_dir) - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(linux_arm64)/ + sudo ls -lRt $(linux_se_arm64)/ + cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + workingDirectory: $(root_dir) + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_se_arm64)/ - # cp NOTICE.txt $(linux_arm64)/ - # mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy - # cp $(linux_arm64)/azcopy pkgDir/usr/bin/ - # displayName: 'Copy required files' - - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # $(linux_arm64)/azcopy --version - # azcopyOutput=$($(linux_arm64)/azcopy --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + cp NOTICE.txt $(linux_se_arm64)/ + cp NOTICE.txt $(linux_arm64)/ + mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy + cp $(linux_arm64)/azcopy pkgDir/usr/bin/ + displayName: 'Copy required files' + + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy --version + azcopyOutput=$($(linux_arm64)/azcopy --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # # TODO: Add description - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -451,7 +447,7 @@ stages: workingDirectory: $(root_dir) - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" displayName: 'Generate Windows AMD64' env: GOARCH: amd64 @@ -459,7 +455,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" displayName: 'Generate Windows i386' env: GOARCH: 386 @@ -467,7 +463,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" displayName: 'Generate Windows ARM' env: GOARCH: arm @@ -476,7 +472,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" displayName: 'Generate Windows ARM' env: GOARCH: arm64 @@ -490,7 +486,7 @@ stages: condition: succeededOrFailed() - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm displayName: 'Remove arm7 binary from staging directory' - task: PublishBuildArtifacts@1 @@ -505,210 +501,209 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: darwin_amd_dir - # value: '$(work_dir)/azcopy_darwin_amd64' - # - name: darwin_arm_dir - # value: '$(work_dir)/azcopy_darwin_arm64' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # mkdir $(darwin_amd_dir) - # mkdir $(darwin_arm_dir) - # mkdir $(archives) - # displayName: 'Create required directories' - - # - script: | - # CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: darwin_amd_dir + value: '$(work_dir)/azcopy_darwin_amd64' + - name: darwin_arm_dir + value: '$(work_dir)/azcopy_darwin_arm64' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' - # - script: | - # cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - # cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - # displayName: "Copy binaries to staging directory" - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # displayName: "Remove binaries from staging directory" - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy NOTICE file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) + cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) + displayName: "Copy binaries to staging directory" + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + displayName: "Remove binaries from staging directory" + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: '*.rpm, *.deb, *tar.gz' - # DisplayName: 'ESRP CodeSigning azCopy linux' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/mariner - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - # Pattern: '*.rpm, *.deb' - # DisplayName: 'ESRP CodeSigning azCopy mariner' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -740,7 +735,6 @@ stages: echo ${{ parameters.tag }} displayName: 'Tag Name' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: @@ -782,7 +776,7 @@ stages: parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: '*.exe' + Pattern: 'azcopy_windows*' # Validate signed images have md5sum changed - script: | @@ -805,15 +799,15 @@ stages: displayName: 'Extract the generated build version' - script: | - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64) + cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386) cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) - mv $(windows_amd64)/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(windows_arm64)/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(windows_386)/azcopy_windows_386.exe $(windows_386)/azcopy.exe + mv $(windows_amd64)/azcopy_windows_amd64 $(windows_amd64)/azcopy + mv $(windows_arm64)/azcopy_windows_arm64 $(windows_arm64)/azcopy + mv $(windows_386)/azcopy_windows_386 $(windows_386)/azcopy zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . @@ -858,171 +852,170 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy signed mac binary to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: '*.exe' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1030,270 +1023,270 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Change to the directory containing azcopy - # cd $EXTRACT_DIR/azcopy_darwin_amd64 - - # # Run the azcopy version command - # ./azcopy --version - # displayName: 'Install Zip, Extract Files, and Run Version Command' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Change to the directory containing azcopy + cd $EXTRACT_DIR/azcopy_darwin_amd64 + + # Run the azcopy version command + ./azcopy --version + displayName: 'Install Zip, Extract Files, and Run Version Command' - job: Set_7 timeoutInMinutes: 120 @@ -1361,14 +1354,14 @@ stages: # Change to the directory containing azcopy Set-Location -Path $extractDir - # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy.exe")) { - Write-Error "azcopy.exe not found in extraction directory: $extractDir" + # Verify that azcopy exists + if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { + Write-Error "azcopy not found in extraction directory: $extractDir" exit 1 } # Run the azcopy version command - .\azcopy_windows_amd64\azcopy.exe --version + .\azcopy_windows_amd64\azcopy --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1422,6 +1415,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + cp NOTICE.txt azCopy-binaries-signed/ sudo apt-get install -y zip zip -r $(Build.ArtifactStagingDirectory)/drop.zip . cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . @@ -1494,7 +1488,6 @@ stages: echo ${{ parameters.tag }} displayName: 'Tag Name' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Linux Build Artifacts' inputs: @@ -1712,7 +1705,6 @@ stages: python3 --version displayName: 'Installing Python' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: From 04a32306be85a6d58ec72b672b19486a533fa589 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 12:36:05 +0530 Subject: [PATCH 310/513] test darwin --- release-pipelines.yml | 927 +++++++++++++++++++++--------------------- 1 file changed, 456 insertions(+), 471 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b70bb497f..bfe10b842 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,462 +44,462 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_amd64 - value: $(work_dir)/linux_amd64 - - name: linux_se_amd64 - value: $(work_dir)/linux_se_amd64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_amd64) - mkdir $(linux_se_amd64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(linux_amd64)/ - sudo ls -lRt $(linux_se_amd64)/ - cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - workingDirectory: $(root_dir) - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_amd64 + # value: $(work_dir)/linux_amd64 + # - name: linux_se_amd64 + # value: $(work_dir)/linux_se_amd64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_amd64) + # mkdir $(linux_se_amd64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_amd64)/ + # sudo ls -lRt $(linux_se_amd64)/ + # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + # workingDirectory: $(root_dir) + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy - mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy - cp $(linux_amd64)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64)/ - cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files for packaging' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - - script: | - export GOARCH=amd64 - export GOOS=linux - $(linux_amd64)/azcopy --version - azcopyOutput=$($(linux_amd64)/azcopy --version) - - # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy + # mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy + # cp $(linux_amd64)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64)/ + # cp NOTICE.txt $(linux_se_amd64)/ + # displayName: 'Copy required files for packaging' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # $(linux_amd64)/azcopy --version + # azcopyOutput=$($(linux_amd64)/azcopy --version) + + # # Run azcopy --version and capture the output + # azcopyOutput=$($(linux_amd64)/azcopy --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64)/*' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64)/*' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - name: linux_arm64 - value: $(work_dir)/linux_arm64 - - name: linux_se_arm64 - value: $(work_dir)/linux_se_arm64 - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - mkdir $(linux_arm64) - mkdir $(linux_se_arm64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + # - name: linux_arm64 + # value: $(work_dir)/linux_arm64 + # - name: linux_se_arm64 + # value: $(work_dir)/linux_se_arm64 + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # mkdir $(linux_arm64) + # mkdir $(linux_se_arm64) + # mkdir $(archives) + # displayName: "Create required directories" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(linux_arm64)/ - sudo ls -lRt $(linux_se_arm64)/ - cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - workingDirectory: $(root_dir) - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(linux_arm64)/ + # sudo ls -lRt $(linux_se_arm64)/ + # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) + # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + # workingDirectory: $(root_dir) + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp NOTICE.txt $(linux_se_arm64)/ - cp NOTICE.txt $(linux_arm64)/ - mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy - cp $(linux_arm64)/azcopy pkgDir/usr/bin/ - displayName: 'Copy required files' - - - script: | - export GOARCH=arm64 - export GOOS=linux - $(linux_arm64)/azcopy --version - azcopyOutput=$($(linux_arm64)/azcopy --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_se_arm64)/ + # cp NOTICE.txt $(linux_arm64)/ + # mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy + # cp $(linux_arm64)/azcopy pkgDir/usr/bin/ + # displayName: 'Copy required files' + + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # $(linux_arm64)/azcopy --version + # azcopyOutput=$($(linux_arm64)/azcopy --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64)/*' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64)/*' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - displayName: 'Remove arm7 binary from staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + # displayName: 'Remove arm7 binary from staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_4 timeoutInMinutes: 360 @@ -537,47 +537,32 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | - mkdir $(darwin_amd_dir) - mkdir $(darwin_arm_dir) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy" displayName: 'Generate MacOS Build with AMD64' - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy" displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - script: | - cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - displayName: "Copy binaries to staging directory" - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - displayName: "Remove binaries from staging directory" - task: Bash@3 displayName: 'Extract AZCopy version' inputs: targetType: 'inline' script: | - azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + azcopy_version=$($(darwin_amd_dir)/azcopy --version | awk '{print $3}' | cut -d'-' -f1) echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + - script: | + echo "##vso[task.setvariable variable=darwin_amd_dir]$(darwin_amd_dir)_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$(darwin_arm_dir)_$(azcopy_version)" + mkdir $(darwin_amd_dir) + mkdir $(darwin_arm_dir) + mkdir $(archives) + displayName: 'Create required directories' + - script: | cp NOTICE.txt $(darwin_amd_dir) cp NOTICE.txt $(darwin_arm_dir) - mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy displayName: 'Copy NOTICE file' - task: ArchiveFiles@2 @@ -816,21 +801,21 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit AMD Build' inputs: - rootFolderOrFile: '$(windows_amd64)' + rootFolderOrFile: '$(windows_amd64)/*' archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit ARM Build' inputs: - rootFolderOrFile: '$(windows_arm64)' + rootFolderOrFile: '$(windows_arm64)/*' archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: - rootFolderOrFile: '$(windows_386)' + rootFolderOrFile: '$(windows_386)/*' archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' continueOnError: true @@ -1428,7 +1413,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive all binaries to drop.zip' inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/*' archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' continueOnError: true From 4afdf1f279af535b76e5b4ae510d65610371cbfd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 13:13:57 +0530 Subject: [PATCH 311/513] test darwin --- release-pipelines.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index bfe10b842..414824999 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -516,10 +516,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: darwin_amd_dir - value: '$(work_dir)/azcopy_darwin_amd64' - - name: darwin_arm_dir - value: '$(work_dir)/azcopy_darwin_arm64' - name: archives value: $(root_dir)/archives @@ -537,11 +533,11 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy" + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" displayName: 'Generate MacOS Build with AMD64' - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy" + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" displayName: 'Test Cross-compiled MacOS Build with ARM64' - task: Bash@3 @@ -549,12 +545,16 @@ stages: inputs: targetType: 'inline' script: | - azcopy_version=$($(darwin_amd_dir)/azcopy --version | awk '{print $3}' | cut -d'-' -f1) + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - script: | - echo "##vso[task.setvariable variable=darwin_amd_dir]$(darwin_amd_dir)_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$(darwin_arm_dir)_$(azcopy_version)" + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + mkdir $(darwin_amd_dir) mkdir $(darwin_arm_dir) mkdir $(archives) @@ -563,6 +563,8 @@ stages: - script: | cp NOTICE.txt $(darwin_amd_dir) cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy displayName: 'Copy NOTICE file' - task: ArchiveFiles@2 From 81b441c8c79938b1ab833dd5eb79550b7e7f0c08 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 13:23:14 +0530 Subject: [PATCH 312/513] test darwin --- release-pipelines.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 414824999..44a53e4fb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -549,15 +549,15 @@ stages: echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + darwin_amd_dir="${work_dir}/azcopy_darwin_amd64_${azcopy_version}" echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + darwin_arm_dir="${work_dir}/azcopy_darwin_arm64_${azcopy_version}" echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - - mkdir $(darwin_amd_dir) - mkdir $(darwin_arm_dir) - mkdir $(archives) + + mkdir -p $(darwin_amd_dir) + mkdir -p $(darwin_arm_dir) + mkdir -p $(archives) displayName: 'Create required directories' - script: | @@ -786,6 +786,9 @@ stages: displayName: 'Extract the generated build version' - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64) cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64) cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386) From 917b6b879db1d783acdfa7f6fd3ead1a5e4bf6d1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 13:30:18 +0530 Subject: [PATCH 313/513] test darwin --- release-pipelines.yml | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 44a53e4fb..9f60f03f8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -555,8 +555,8 @@ stages: darwin_arm_dir="${work_dir}/azcopy_darwin_arm64_${azcopy_version}" echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $(darwin_amd_dir) - mkdir -p $(darwin_arm_dir) + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir mkdir -p $(archives) displayName: 'Create required directories' @@ -708,12 +708,6 @@ stages: - group: AZCOPY_ESRP_SECRET - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: windows_amd64 - value: '$(work_dir)/azcopy_windows_amd64' - - name: windows_arm64 - value: '$(work_dir)/azcopy_windows_arm64' - - name: windows_386 - value: '$(work_dir)/azcopy_windows_386' - name: archives value: '$(work_dir)/archives' @@ -739,13 +733,6 @@ stages: sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* displayName: 'List Artifacts' - - - script: | - mkdir -p $(windows_amd64) - mkdir -p $(windows_arm64) - mkdir -p $(windows_386) - mkdir -p $(archives) - displayName: 'Create required directories' - script: | sudo apt-get clean @@ -787,17 +774,25 @@ stages: - script: | windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386) + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir $(windows_amd64) + mkdir $(windows_arm64) + mkdir $(windows_386) + mkdir -p $(archives) + + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) - mv $(windows_amd64)/azcopy_windows_amd64 $(windows_amd64)/azcopy - mv $(windows_arm64)/azcopy_windows_arm64 $(windows_arm64)/azcopy - mv $(windows_386)/azcopy_windows_386 $(windows_386)/azcopy zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . From 62cd57bc3c5ca9a9c4f20c3954a85d57a6b19879 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 13:36:15 +0530 Subject: [PATCH 314/513] test darwin --- release-pipelines.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9f60f03f8..785f67489 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -549,10 +549,10 @@ stages: echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - script: | - darwin_amd_dir="${work_dir}/azcopy_darwin_amd64_${azcopy_version}" + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - darwin_arm_dir="${work_dir}/azcopy_darwin_arm64_${azcopy_version}" + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" mkdir -p $darwin_amd_dir @@ -782,11 +782,13 @@ stages: windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" echo "##vso[task.setvariable variable=windows_386]$windows_386" - mkdir $(windows_amd64) - mkdir $(windows_arm64) - mkdir $(windows_386) + mkdir $windows_amd64 + mkdir $windows_arm64 + mkdir $windows_386 mkdir -p $(archives) + displayName: 'Create required directories' + - script: | mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy @@ -801,21 +803,21 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit AMD Build' inputs: - rootFolderOrFile: '$(windows_amd64)/*' + rootFolderOrFile: '$(windows_amd64)' archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 64 bit ARM Build' inputs: - rootFolderOrFile: '$(windows_arm64)/*' + rootFolderOrFile: '$(windows_arm64)' archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 displayName: 'Archive Windows 32 bit Build' inputs: - rootFolderOrFile: '$(windows_386)/*' + rootFolderOrFile: '$(windows_386)' archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' continueOnError: true From 53c71e3d156d887c0ea5b1300f59b54d16aa9503 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 13:59:30 +0530 Subject: [PATCH 315/513] test zips --- release-pipelines.yml | 880 +++++++++++++++++++++--------------------- 1 file changed, 438 insertions(+), 442 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 785f67489..dfde560fa 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,462 +44,458 @@ pr: none stages: - stage: BuildArtifacts jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_amd64 - # value: $(work_dir)/linux_amd64 - # - name: linux_se_amd64 - # value: $(work_dir)/linux_se_amd64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_amd64) - # mkdir $(linux_se_amd64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_amd64)/ - # sudo ls -lRt $(linux_se_amd64)/ - # cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - # workingDirectory: $(root_dir) - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy - # mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy - # cp $(linux_amd64)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64)/ - # cp NOTICE.txt $(linux_se_amd64)/ - # displayName: 'Copy required files for packaging' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # $(linux_amd64)/azcopy --version - # azcopyOutput=$($(linux_amd64)/azcopy --version) - - # # Run azcopy --version and capture the output - # azcopyOutput=$($(linux_amd64)/azcopy --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_amd64_dir="${work_dir}/azcopy_linux_amd64_${azcopy_version}" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="${work_dir}/azcopy_linux_amd64_${azcopy_version}" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + mkdir $linux_amd64_dir + mkdir $linux_se_amd64_dir + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + displayName: 'Copy required files for packaging' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64)/*' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64)/*' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - # - name: linux_arm64 - # value: $(work_dir)/linux_arm64 - # - name: linux_se_arm64 - # value: $(work_dir)/linux_se_arm64 - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # mkdir $(linux_arm64) - # mkdir $(linux_se_arm64) - # mkdir $(archives) - # displayName: "Create required directories" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + value: $(work_dir)/linux_se_arm64 + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(linux_arm64)/ - # sudo ls -lRt $(linux_se_arm64)/ - # cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - # cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - # workingDirectory: $(root_dir) - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_se_arm64)/ - # cp NOTICE.txt $(linux_arm64)/ - # mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy - # cp $(linux_arm64)/azcopy pkgDir/usr/bin/ - # displayName: 'Copy required files' - - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # $(linux_arm64)/azcopy --version - # azcopyOutput=$($(linux_arm64)/azcopy --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + - script: | + export GOARCH=arm64 + export GOOS=linux + $(linux_arm64)/azcopy --version + azcopyOutput=$($(linux_arm64)/azcopy --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_arm64_dir="${work_dir}/azcopy_linux_arm64_${azcopy_version}" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="${work_dir}/azcopy_linux_arm64_${azcopy_version}" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir $linux_arm64_dir + mkdir $linux_se_arm64_dir + mkdir $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64)/*' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64)/*' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - # displayName: 'Remove arm7 binary from staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_4 timeoutInMinutes: 360 From 7c29cac7f4bd99c524277ba2dcf833e184f0414b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 14:01:43 +0530 Subject: [PATCH 316/513] test zips --- release-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index dfde560fa..2eb227e1e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -252,7 +252,6 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - name: archives value: $(root_dir)/archives - value: $(work_dir)/linux_se_arm64 steps: #TODO: remove this after merging to main From 42cae7eb71403e39e1eb20cda9cc7e3f2315c918 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 14:48:40 +0530 Subject: [PATCH 317/513] test zips --- release-pipelines.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2eb227e1e..7dc688b41 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -149,15 +149,15 @@ stages: displayName: 'Verify and extract the generated build version' - script: | - linux_amd64_dir="${work_dir}/azcopy_linux_amd64_${azcopy_version}" + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - linux_se_amd64_dir="${work_dir}/azcopy_linux_amd64_${azcopy_version}" + linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - mkdir $linux_amd64_dir - mkdir $linux_se_amd64_dir - mkdir $(archives) + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) displayName: 'Create required directories' - script: | @@ -336,15 +336,15 @@ stages: displayName: 'Verify and extract the generated build version' - script: | - linux_arm64_dir="${work_dir}/azcopy_linux_arm64_${azcopy_version}" + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - linux_se_arm64_dir="${work_dir}/azcopy_linux_arm64_${azcopy_version}" + linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - mkdir $linux_arm64_dir - mkdir $linux_se_arm64_dir - mkdir $(archives) + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) displayName: 'Create required directories' - script: | From b0a9e82a7eba7a7a1e92afd9d97fc251f3c6638b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 26 Jun 2024 15:10:28 +0530 Subject: [PATCH 318/513] test zips --- release-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7dc688b41..ce65f765c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -314,8 +314,7 @@ stages: - script: | export GOARCH=arm64 export GOOS=linux - $(linux_arm64)/azcopy --version - azcopyOutput=$($(linux_arm64)/azcopy --version) + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) # Extract the version part using grep and sed if echo "$azcopyOutput" | grep -q "azcopy version"; then From c621b97d290609a7dda0f711e0316999ca9f824e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 27 Jun 2024 09:29:28 +0530 Subject: [PATCH 319/513] test zips --- release-pipelines.yml | 76 +++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ce65f765c..a909cf733 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -359,7 +359,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ + fpm -s dir -t deb -n azcopy --rpm-digest -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -1057,6 +1057,7 @@ stages: - checkout: none - script: | sudo apt update + sudo apt-get install libsecret-1-dev sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" @@ -1096,6 +1097,7 @@ stages: - checkout: none - script: | sudo yum update -y + sudo yum install libsecret sudo yum install git -y displayName: 'Install Git' @@ -1136,6 +1138,7 @@ stages: - checkout: none - script: | sudo yum update -y + sudo yum install libsecret sudo yum install git -y displayName: 'Install Git' @@ -1182,6 +1185,7 @@ stages: - checkout: none - script: | sudo tdnf update -y + sudo tdnf install libsecret sudo tdnf install git -y displayName: 'Install Git' @@ -1262,8 +1266,16 @@ stages: # Extract the zip file unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Change to the directory containing azcopy - cd $EXTRACT_DIR/azcopy_darwin_amd64 + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi # Run the azcopy version command ./azcopy --version @@ -1335,14 +1347,28 @@ stages: # Change to the directory containing azcopy Set-Location -Path $extractDir - # Verify that azcopy exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { - Write-Error "azcopy not found in extraction directory: $extractDir" - exit 1 + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 } - # Run the azcopy version command - .\azcopy_windows_amd64\azcopy --version + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + } + + & $azcopyPath --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1622,15 +1648,6 @@ stages: marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" @@ -1645,10 +1662,29 @@ stages: azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) echo "Azcopy ARM RPM ID: $azcopyArmRpm" - while IFS=, read -r distro fuseArchType repoName releaseName; do + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + echo "Package $marinerAmdRpmFile contains 'preview'." + while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv) + done < <(tail -n +5 ../packages.csv) + else + echo "Package $package_name1 does not contain 'preview'." + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv | head -n -2) + fi + displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ From 6dc09b456743b5629941e29f1d08a05f0a53a4ad Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 27 Jun 2024 09:29:28 +0530 Subject: [PATCH 320/513] test zips --- packages.csv | 8 +++-- release-pipelines.yml | 76 +++++++++++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/packages.csv b/packages.csv index 120605c75..e7cd95d29 100644 --- a/packages.csv +++ b/packages.csv @@ -1,5 +1,7 @@ # Do not remove these comments # Format of the file : ,,, +Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, +Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, Ubuntu-18.04,azcopyAmdDeb,microsoft-ubuntu-bionic-prod-apt,bionic Ubuntu-20.04,azcopyAmdDeb,microsoft-ubuntu-focal-prod-apt,focal Ubuntu-22.04,azcopyAmdDeb,microsoft-ubuntu-jammy-prod-apt,jammy @@ -20,7 +22,7 @@ RHEL-9.0,azcopyArmRpm,microsoft-rhel9.0-prod-yum, CentOS-7.0,azcopyAmdRpm,microsoft-centos7-prod-yum, CentOS-8.0,azcopyAmdRpm,microsoft-centos8-prod-yum, SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, -Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, -Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, \ No newline at end of file +Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, +Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-preview-Microsoft-x86_64-yum, +Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-preview-Microsoft-aarch64-yum, \ No newline at end of file diff --git a/release-pipelines.yml b/release-pipelines.yml index ce65f765c..38fd15f74 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -359,7 +359,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -1057,6 +1057,7 @@ stages: - checkout: none - script: | sudo apt update + sudo apt-get install libsecret-1-dev sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" @@ -1096,6 +1097,7 @@ stages: - checkout: none - script: | sudo yum update -y + sudo yum install libsecret sudo yum install git -y displayName: 'Install Git' @@ -1136,6 +1138,7 @@ stages: - checkout: none - script: | sudo yum update -y + sudo yum install libsecret sudo yum install git -y displayName: 'Install Git' @@ -1182,6 +1185,7 @@ stages: - checkout: none - script: | sudo tdnf update -y + sudo tdnf install libsecret sudo tdnf install git -y displayName: 'Install Git' @@ -1262,8 +1266,16 @@ stages: # Extract the zip file unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Change to the directory containing azcopy - cd $EXTRACT_DIR/azcopy_darwin_amd64 + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi # Run the azcopy version command ./azcopy --version @@ -1335,14 +1347,28 @@ stages: # Change to the directory containing azcopy Set-Location -Path $extractDir - # Verify that azcopy exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { - Write-Error "azcopy not found in extraction directory: $extractDir" - exit 1 + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 } - # Run the azcopy version command - .\azcopy_windows_amd64\azcopy --version + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + } + + & $azcopyPath --version displayName: 'Extract Files and Run Version Command on Windows' # TestArtifacts ends here @@ -1622,15 +1648,6 @@ stages: marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` echo "Azcopy AMD DEB ID: $azcopyAmdDeb" @@ -1645,10 +1662,29 @@ stages: azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) echo "Azcopy ARM RPM ID: $azcopyArmRpm" - while IFS=, read -r distro fuseArchType repoName releaseName; do + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + echo "Package $marinerAmdRpmFile contains 'preview'." + while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv) + done < <(tail -n +5 ../packages.csv) + else + echo "Package $package_name1 does not contain 'preview'." + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv | head -n -2) + fi + displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ From d701629d07ab54107d3449fb89583cd22102d320 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 27 Jun 2024 16:06:59 +0530 Subject: [PATCH 321/513] test fips deb and rpms --- release-pipelines.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7c304edd9..60d3a9585 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1185,9 +1185,19 @@ stages: - checkout: none - script: | sudo tdnf update -y - sudo tdnf install libsecret sudo tdnf install git -y - displayName: 'Install Git' + displayName: 'Install dependencies' + + - script: | + sudo tdnf install wget tar gcc make glib2-devel gnome-keyring-devel + wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + tar -xf libsecret-0.20.4.tar.xz + cd libsecret-0.20.4 + ./configure + make + sudo make install + sudo ldconfig + diaplayName: 'Install libsecret' - template: setup/test_artifacts.yml parameters: @@ -1210,6 +1220,7 @@ stages: azcopy --version azcopy --help displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - job: Set_6 timeoutInMinutes: 360 From 7db50b19d5757c143f9a1bd6c4a980dcfb1888dd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 27 Jun 2024 16:08:07 +0530 Subject: [PATCH 322/513] test fips deb and rpms --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 60d3a9585..07b141b6e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1197,7 +1197,7 @@ stages: make sudo make install sudo ldconfig - diaplayName: 'Install libsecret' + displayName: 'Install libsecret' - template: setup/test_artifacts.yml parameters: From 1d2150385576b947e2573cac656e0bb85adc0e45 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 27 Jun 2024 17:45:24 +0530 Subject: [PATCH 323/513] test fips deb and rpms --- release-pipelines.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 07b141b6e..511b1f461 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1057,8 +1057,8 @@ stages: - checkout: none - script: | sudo apt update - sudo apt-get install libsecret-1-dev - sudo apt --fix-broken install + sudo apt-get install libsecret-1-dev -y + sudo apt --fix-broken install -y ldd --version displayName: "GLIBC Version" @@ -1097,7 +1097,7 @@ stages: - checkout: none - script: | sudo yum update -y - sudo yum install libsecret + sudo yum install libsecret -y sudo yum install git -y displayName: 'Install Git' @@ -1138,7 +1138,7 @@ stages: - checkout: none - script: | sudo yum update -y - sudo yum install libsecret + sudo yum install libsecret -y sudo yum install git -y displayName: 'Install Git' @@ -1185,19 +1185,20 @@ stages: - checkout: none - script: | sudo tdnf update -y + sudo tdnf install libsecret -y sudo tdnf install git -y displayName: 'Install dependencies' - - script: | - sudo tdnf install wget tar gcc make glib2-devel gnome-keyring-devel - wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - tar -xf libsecret-0.20.4.tar.xz - cd libsecret-0.20.4 - ./configure - make - sudo make install - sudo ldconfig - displayName: 'Install libsecret' + # - script: | + # sudo tdnf install wget tar gcc make glib2-devel gnome-keyring-devel + # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + # tar -xf libsecret-0.20.4.tar.xz + # cd libsecret-0.20.4 + # ./configure + # make + # sudo make install + # sudo ldconfig + # displayName: 'Install libsecret' - template: setup/test_artifacts.yml parameters: From 9121937104789c5738f963b0dc3a91cf996d0c7b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 27 Jun 2024 17:45:24 +0530 Subject: [PATCH 324/513] test fips deb and rpms --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 07b141b6e..ef346b368 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1057,8 +1057,8 @@ stages: - checkout: none - script: | sudo apt update - sudo apt-get install libsecret-1-dev - sudo apt --fix-broken install + sudo apt-get install libsecret-1-dev -y + sudo apt --fix-broken install -y ldd --version displayName: "GLIBC Version" @@ -1097,7 +1097,7 @@ stages: - checkout: none - script: | sudo yum update -y - sudo yum install libsecret + sudo yum install libsecret -y sudo yum install git -y displayName: 'Install Git' @@ -1138,7 +1138,7 @@ stages: - checkout: none - script: | sudo yum update -y - sudo yum install libsecret + sudo yum install libsecret -y sudo yum install git -y displayName: 'Install Git' From c2dcf311cd761d7dd7d7ffdfba69055d7ba73d5b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 10:41:51 +0530 Subject: [PATCH 325/513] test mariner --- release-pipelines.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 511b1f461..2e8419a7b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1185,20 +1185,24 @@ stages: - checkout: none - script: | sudo tdnf update -y - sudo tdnf install libsecret -y sudo tdnf install git -y displayName: 'Install dependencies' - # - script: | - # sudo tdnf install wget tar gcc make glib2-devel gnome-keyring-devel - # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - # tar -xf libsecret-0.20.4.tar.xz - # cd libsecret-0.20.4 - # ./configure - # make - # sudo make install - # sudo ldconfig - # displayName: 'Install libsecret' + - script: | + # Update package list + sudo tdnf update + + # Install libsecret + sudo tdnf install libsecret -y + + # Verify installation + if ldconfig -p | grep -q libsecret-1.so.0; then + echo "libsecret installed successfully." + else + echo "libsecret installation failed or libsecret-1.so.0 not found." + exit 1 + fi + displayName: 'Install libsecret' - template: setup/test_artifacts.yml parameters: From 238ff8d6656d0fe2936e281f177d3a8c09f52c2c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 10:59:09 +0530 Subject: [PATCH 326/513] test mariner --- release-pipelines.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2e8419a7b..1670bc94e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1688,33 +1688,33 @@ stages: if [[ $marinerAmdRpmFile == *"preview"* ]]; then echo "Package $marinerAmdRpmFile contains 'preview'." - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +5 ../packages.csv) + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +5 ../packages.csv) else echo "Package $package_name1 does not contain 'preview'." - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv | head -n -2) + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv | head -n -2) fi displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - script: | - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv) - displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + # - script: | + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion dependsOn: ReleaseArtifacts From 9754d1b6c5e06775d8c7195bc8ddcf94b3616f49 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 11:38:12 +0530 Subject: [PATCH 327/513] test mariner publish --- release-pipelines.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1670bc94e..eb9e212a0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1187,22 +1187,6 @@ stages: sudo tdnf update -y sudo tdnf install git -y displayName: 'Install dependencies' - - - script: | - # Update package list - sudo tdnf update - - # Install libsecret - sudo tdnf install libsecret -y - - # Verify installation - if ldconfig -p | grep -q libsecret-1.so.0; then - echo "libsecret installed successfully." - else - echo "libsecret installation failed or libsecret-1.so.0 not found." - exit 1 - fi - displayName: 'Install libsecret' - template: setup/test_artifacts.yml parameters: @@ -1217,6 +1201,7 @@ stages: - script: | sudo rpm -qip azcopy*x86_64.rpm + sudo tdnf install gcc build-essential -y sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner From 82a73198e0d3b08a0930aab2c400d91c6dc8f487 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 16:37:43 +0530 Subject: [PATCH 328/513] test mariner publish --- release-pipelines.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index eb9e212a0..ad777636f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1187,6 +1187,18 @@ stages: sudo tdnf update -y sudo tdnf install git -y displayName: 'Install dependencies' + + - script: | + sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make + wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + tar -xf libsecret-0.20.4.tar.xz + cd libsecret-0.20.4 + ./configure + make + sudo make install + ldconfig -p | grep libsecret + displayName: 'Install libsecret' + - template: setup/test_artifacts.yml parameters: @@ -1210,7 +1222,6 @@ stages: azcopy --version azcopy --help displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - job: Set_6 timeoutInMinutes: 360 From 6d5d94c057557385da2b6eef7463c53da8cd244f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 18:54:12 +0530 Subject: [PATCH 329/513] test mariner installations --- release-pipelines.yml | 3320 +++++++++++++++++++++-------------------- 1 file changed, 1691 insertions(+), 1629 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ad777636f..a0ad1b51f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,1720 +44,1782 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Set_1 + - job: Set_5 timeoutInMinutes: 120 strategy: matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + pool: - name: "blobfuse-ubuntu-pool" + name: "blobfuse-mariner-pool" demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives + - ImageOverride -equals $(AgentName) steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done + - checkout: none - script: | - ldd --version - displayName: "GLIBC Version" + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' + + - script: | - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' + # Install necessary packages + sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' + # Verify gcc and make + gcc --version + make --version - - script: | - sudo ls -lRt $(root_dir)/ - cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" + # Download and extract libsecret + wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + tar -xf libsecret-0.20.4.tar.xz + cd libsecret-0.20.4 - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" + # Configure and compile libsecret + ./configure + if [ $? -ne 0 ]; then + echo "Configuration failed. Check config.log for details." + cat config.log exit 1 fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - displayName: 'Create required directories' + make + if [ $? -ne 0 ]; then + echo "Make failed." + exit 1 + fi - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ - displayName: 'Copy required files for packaging' + sudo make install + if [ $? -ne 0 ]; then + echo "Make install failed." + exit 1 + fi - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' + # Verify installation + ldconfig -p | grep libsecret - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + displayName: 'Install libsecret' + + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # # Run azcopy --version and capture the output + # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # - script: | + # linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + # linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + # mkdir -p $linux_amd64_dir + # mkdir -p $linux_se_amd64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + # cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64_dir)/ + # cp NOTICE.txt $(linux_se_amd64_dir)/ + # displayName: 'Copy required files for packaging' + + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(root_dir)/ - cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # - script: | + # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + # linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + # mkdir -p $linux_arm64_dir + # mkdir -p $linux_se_arm64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_arm64_dir)/ + # cp NOTICE.txt $(linux_se_arm64_dir)/ + # displayName: 'Copy required files for packaging' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - displayName: 'Remove arm7 binary from staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + # displayName: 'Remove arm7 binary from staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy NOTICE file' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy NOTICE file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - stage: SignArtifacts + # dependsOn: BuildArtifacts + # condition: succeeded('BuildArtifacts') + # jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, *tar.gz' - DisplayName: 'ESRP CodeSigning azCopy linux' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: '*.rpm, *.deb, *tar.gz' + # DisplayName: 'ESRP CodeSigning azCopy linux' - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/mariner - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - Pattern: '*.rpm, *.deb' - DisplayName: 'ESRP CodeSigning azCopy mariner' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/mariner + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + # Pattern: '*.rpm, *.deb' + # DisplayName: 'ESRP CodeSigning azCopy mariner' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir $windows_amd64 - mkdir $windows_arm64 - mkdir $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + + # - script: | + # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir $windows_amd64 + # mkdir $windows_arm64 + # mkdir $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac binary to staging directory' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy signed mac binary to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' - - # SignArtifacts end here - - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - script: | - sudo apt update - sudo apt-get install libsecret-1-dev -y - sudo apt --fix-broken install -y - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' + + # # SignArtifacts end here + + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt-get install libsecret-1-dev -y + # sudo apt --fix-broken install -y + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install libsecret -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install libsecret -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install libsecret -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install libsecret -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install dependencies' + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install dependencies' - - script: | - sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make - wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - tar -xf libsecret-0.20.4.tar.xz - cd libsecret-0.20.4 - ./configure - make - sudo make install - ldconfig -p | grep libsecret - displayName: 'Install libsecret' - - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo tdnf install gcc build-essential -y - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi - - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run the azcopy version command - ./azcopy --version - displayName: 'Install Zip, Extract Files, and Run Version Command' + # - script: | + # sudo tdnf install -y binutils gcc gcc-c++ make + # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make + # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + # tar -xf libsecret-0.20.4.tar.xz + # cd libsecret-0.20.4 + # ./configure + # make + # sudo make install + # ldconfig -p | grep libsecret + # displayName: 'Install libsecret' + + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo rpm -qip azcopy*x86_64.rpm + # sudo tdnf install gcc build-essential -y + # sudo rpm -i azcopy*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi + + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # # Create extraction directory + # mkdir -p $EXTRACT_DIR + + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # # Find the directory matching the pattern + # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi + + # # Run the azcopy version command + # ./azcopy --version + # displayName: 'Install Zip, Extract Files, and Run Version Command' - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Find directories matching the pattern - $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # Check if there is exactly one matching directory - if ($matchingDirs.Count -eq 0) { - Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } elseif ($matchingDirs.Count -gt 1) { - Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } - - # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" - if (-Not (Test-Path -Path $azcopyPath)) { - Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - exit 1 - } else { - Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - } - & $azcopyPath --version - displayName: 'Extract Files and Run Version Command on Windows' - - # TestArtifacts ends here + # # Find the zip file matching the pattern + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } + + # $zipFilePath = $zipFile.FullName + + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } + + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir + + # # Find directories matching the pattern + # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # # Check if there is exactly one matching directory + # if ($matchingDirs.Count -eq 0) { + # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } elseif ($matchingDirs.Count -gt 1) { + # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } + + # # Check if the azcopy executable exists in the matching directory + # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + # if (-Not (Test-Path -Path $azcopyPath)) { + # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + # exit 1 + # } else { + # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # } + # & $azcopyPath --version + # displayName: 'Extract Files and Run Version Command on Windows' + + # # TestArtifacts ends here - - stage: ReleaseArtifacts - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseToContainer - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) + # - stage: ReleaseArtifacts + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') + # jobs: + # - job: ReleaseToContainer + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) - variables: - - name: Test_Release_Account_Name - value: 'azcopyprivatedrops' - - name: Release_Account_Name - value: 'azcopyvnextrelease' - - name: Test_Mode - value: ${{ parameters.test_mode }} - - steps: - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # variables: + # - name: Test_Release_Account_Name + # value: 'azcopyprivatedrops' + # - name: Release_Account_Name + # value: 'azcopyvnextrelease' + # - name: Test_Mode + # value: ${{ parameters.test_mode }} + + # steps: + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Linux Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Windows Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Mac Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Binaries Build Artifacts' - inputs: - artifactName: 'azCopy-binaries-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - cp NOTICE.txt azCopy-binaries-signed/ - sudo apt-get install -y zip - zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - task: ArchiveFiles@2 - displayName: 'Archive all binaries to drop.zip' - inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/*' - archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' - continueOnError: true - - - task: Bash@3 - displayName: 'Install Azure CLI' - inputs: - targetType: 'inline' - script: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - - task: AzureCLI@2 - displayName: 'Azure CLI' - inputs: - azureSubscription: 'ESRP KeyVault identity' - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - # Find the current version - raw_version_output=$(azcopy --version) - azcopy_version=$(echo $raw_version_output | awk '{print $3}') - - ls -ltR $(Build.ArtifactStagingDirectory) - - today=$(date +"%Y%m%d") + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Binaries Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # cp NOTICE.txt azCopy-binaries-signed/ + # sudo apt-get install -y zip + # zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + # cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Create drop.zip file and install azcopy deb package' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - task: ArchiveFiles@2 + # displayName: 'Archive all binaries to drop.zip' + # inputs: + # rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/*' + # archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' + # continueOnError: true + + # - task: Bash@3 + # displayName: 'Install Azure CLI' + # inputs: + # targetType: 'inline' + # script: | + # curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + # - task: AzureCLI@2 + # displayName: 'Azure CLI' + # inputs: + # azureSubscription: 'ESRP KeyVault identity' + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # # Find the current version + # raw_version_output=$(azcopy --version) + # azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + # ls -ltR $(Build.ArtifactStagingDirectory) + + # today=$(date +"%Y%m%d") - if [ $(Test_Mode) = "True" ]; then - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - else - container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - fi - echo "Artifacts will be uploaded to: $container_url" + # if [ $(Test_Mode) = "True" ]; then + # container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + # else + # container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + # fi + # echo "Artifacts will be uploaded to: $container_url" - executable_name=azcopy + # executable_name=azcopy - # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - - - job: ReleaseAzcopy - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none + # # Upload the archived builds + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + # - job: ReleaseAzcopy + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Linux Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Windows Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Mac Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip - displayName: 'List Artifacts' - - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + # displayName: 'List Artifacts' + + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - - ${{ if eq(parameters.post_release, true) }}: + # - ${{ if eq(parameters.post_release, true) }}: - # add release tags & push to github - #TODO: update github connection and repo name while merging to main - - task: GithubRelease@1 - inputs: - githubConnection: 'dphulkar-msft-github-rel' - repositoryName: 'dphulkar-msft/azure-storage-azcopy' - action: 'edit' + # # add release tags & push to github + # #TODO: update github connection and repo name while merging to main + # - task: GithubRelease@1 + # inputs: + # githubConnection: 'dphulkar-msft-github-rel' + # repositoryName: 'dphulkar-msft/azure-storage-azcopy' + # action: 'edit' - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' + # target: '$(Build.SourceVersion)' + # tagSource: 'userSpecifiedTag' - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} + # title: ${{ parameters.tag }} + # tag: ${{ parameters.tag }} - assets: | - $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace - - - ${{ if eq(parameters.publish_artifacts, true) }}: - - stage: PublishArtifacts - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: PublishArtifacts - timeoutInMinutes: 120 - pool: - vmImage: 'ubuntu-22.04' - variables: - - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # assets: | + # $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + # $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + # $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + # changeLogCompareToRelease: 'lastFullRelease' + # changeLogType: 'commitBased' + # isDraft: ${{ parameters.draft }} + # isPreRelease: ${{ parameters.prerelease }} + # assetUploadMode: replace + + # - ${{ if eq(parameters.publish_artifacts, true) }}: + # - stage: PublishArtifacts + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: PublishArtifacts + # timeoutInMinutes: 120 + # pool: + # vmImage: 'ubuntu-22.04' + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - checkout: none - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'DevExGlobalFeed' - displayName: 'Connect to PMC artifact' - - - script: | - pip install pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - - # download artifacts that need to be published - - task: DownloadBuildArtifacts@0 - displayName: 'Download Signed Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # steps: + # - checkout: none + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'DevExGlobalFeed' + # displayName: 'Connect to PMC artifact' + + # - script: | + # pip install pmc-cli + # displayName: 'Install pmc-cli' + + # - task: DownloadSecureFile@1 + # name: pmcCertificate + # displayName: 'Download pmc pem file' + # inputs: + # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + # - task: DownloadSecureFile@1 + # name: settings + # displayName: 'Download settings.toml file' + # inputs: + # secureFile: 'settings.toml' + + # - script: | + # pmc --version + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + # if [ $? -ne 0 ]; then + # exit 1 + # fi + # displayName: 'Test PMC installation' + + # # download artifacts that need to be published + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Signed Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - cd mariner - arm64file=$(ls azcopy*.arm64.rpm) - amd64file=$(ls azcopy*.x86_64.rpm) - marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" - marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" - mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - ls -lRt - mv azcopy*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - displayName: 'List Artifacts' - - - script: | - mv azCopy-linux-signed/*tar.gz . - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed - displayName: 'Upload files' - workingDirectory: $(Build.ArtifactStagingDirectory) - - # replace org name while merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv - cat ./packages.csv - displayName: 'Fetch packages.csv' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - # Find the Mariner RPM file and upload - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - - azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + # - script: | + # cd mariner + # arm64file=$(ls azcopy*.arm64.rpm) + # amd64file=$(ls azcopy*.x86_64.rpm) + # marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + # marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + # mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + # mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + # ls -lRt + # mv azcopy*.rpm .. + # cd .. + # rm -r mariner/ + # displayName: 'Rename Mariner binaries' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # displayName: 'List Artifacts' + + # - script: | + # mv azCopy-linux-signed/*tar.gz . + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + # displayName: 'Upload files' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # # replace org name while merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + # cat ./packages.csv + # displayName: 'Fetch packages.csv' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # # Find the Mariner RPM file and upload + # marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + # marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + # echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + # azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + # echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + # azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + # echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" - azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - echo "Azcopy ARM DEB ID: $azcopyArmDeb" - - azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') - azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - echo "Azcopy ARM RPM ID: $azcopyArmRpm" - - # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - - if [[ $marinerAmdRpmFile == *"preview"* ]]; then - echo "Package $marinerAmdRpmFile contains 'preview'." - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +5 ../packages.csv) - else - echo "Package $package_name1 does not contain 'preview'." - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +3 ../packages.csv | head -n -2) - fi - - displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - # - script: | - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Publish the repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + # echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + # azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') + # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + # echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # # Find the Mariner RPM file and upload + # marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + # marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + # marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + # echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + # if [[ $marinerAmdRpmFile == *"preview"* ]]; then + # echo "Package $marinerAmdRpmFile contains 'preview'." + # # while IFS=, read -r distro fuseArchType repoName releaseName; do + # # echo "Uploading packages for $distro" + # # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # # done < <(tail -n +5 ../packages.csv) + # else + # echo "Package $package_name1 does not contain 'preview'." + # # while IFS=, read -r distro fuseArchType repoName releaseName; do + # # echo "Uploading packages for $distro" + # # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # # done < <(tail -n +3 ../packages.csv | head -n -2) + # fi + + # displayName: 'Add uploaded packages to repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + # # - script: | + # # while IFS=, read -r distro fuseArchType repoName releaseName; do + # # if [[ $fuseArchType == *"Arm"* ]] + # # then + # # echo "Skipping for ARM type on $distro" + # # else + # # echo "Publishing for $distro" + # # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # # fi + # # done < <(tail -n +3 ../packages.csv) + # # displayName: 'Publish the repository' + # # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: UpdateVersion - pool: - vmImage: 'ubuntu-20.04' - variables: - - group: AZCOPY_TESTS_VAR - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - stage: UpdateLatestVersion + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: UpdateVersion + # pool: + # vmImage: 'ubuntu-20.04' + # variables: + # - group: AZCOPY_TESTS_VAR + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - checkout: none - - - ${{ if eq(parameters.update_version, true) }}: - - script: | - sudo apt-get install python3 -y - python3 --version - displayName: 'Installing Python' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # steps: + # - checkout: none + + # - ${{ if eq(parameters.update_version, true) }}: + # - script: | + # sudo apt-get install python3 -y + # python3 --version + # displayName: 'Installing Python' + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - # install azcopy - - script: | - cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - ls azcopy*.arm64.deb - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - azcopy --version - displayName: 'Installing azcopy' + # # install azcopy + # - script: | + # cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + # ls azcopy*.arm64.deb + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # azcopy --version + # displayName: 'Installing azcopy' - # TODO: change org name while merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py - ls -l - displayName: 'Getting Python script' - workingDirectory: $(root_dir) - - - script: | - pip install azure-storage-blob - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" - displayName: 'Updating version number' - workingDirectory: $(root_dir) + # # TODO: change org name while merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + # ls -l + # displayName: 'Getting Python script' + # workingDirectory: $(root_dir) + + # - script: | + # pip install azure-storage-blob + # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + # displayName: 'Updating version number' + # workingDirectory: $(root_dir) From 0c89b720460361a6622525a64791e7b4b0389c71 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 19:11:47 +0530 Subject: [PATCH 330/513] test mariner installations --- release-pipelines.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index a0ad1b51f..be3c9c628 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -70,16 +70,25 @@ stages: # Install necessary packages sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake + sudo dnf install binutils + + export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib # Verify gcc and make gcc --version make --version + df -h + # Download and extract libsecret wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz tar -xf libsecret-0.20.4.tar.xz cd libsecret-0.20.4 + echo $PATH + echo $LD_LIBRARY_PATH + # Configure and compile libsecret ./configure if [ $? -ne 0 ]; then From 7b60afbc0f610178efc0f1326881bda6077aea3f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 19:39:22 +0530 Subject: [PATCH 331/513] test mariner installations --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index be3c9c628..696eaf037 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -70,7 +70,8 @@ stages: # Install necessary packages sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake - sudo dnf install binutils + sudo tdnf install binutils -y + sudo tdnf install libsecret -y export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib From 29015ea161838cf26efa6240d35f641432cb127d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 19:59:04 +0530 Subject: [PATCH 332/513] test mariner installations --- release-pipelines.yml | 3469 +++++++++++++++++++++-------------------- 1 file changed, 1770 insertions(+), 1699 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 696eaf037..a6fe25478 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,1792 +44,1863 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Set_5 + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install dependencies' + + # - script: | + + # # Install necessary packages + # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake + # sudo tdnf install binutils -y + # sudo tdnf install libsecret -y + + # export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + # export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib + + # # Verify gcc and make + # gcc --version + # make --version + + # df -h + + # # Download and extract libsecret + # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + # tar -xf libsecret-0.20.4.tar.xz + # cd libsecret-0.20.4 + + # echo $PATH + # echo $LD_LIBRARY_PATH + + # # Configure and compile libsecret + # ./configure + # if [ $? -ne 0 ]; then + # echo "Configuration failed. Check config.log for details." + # cat config.log + # exit 1 + # fi + + # make + # if [ $? -ne 0 ]; then + # echo "Make failed." + # exit 1 + # fi + + # sudo make install + # if [ $? -ne 0 ]; then + # echo "Make install failed." + # exit 1 + # fi + + # # Verify installation + # ldconfig -p | grep libsecret + + # displayName: 'Install libsecret' + + - job: Set_1 timeoutInMinutes: 120 strategy: matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - name: "blobfuse-mariner-pool" + name: "blobfuse-ubuntu-pool" demands: - - ImageOverride -equals $(AgentName) + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives steps: - - checkout: none + #TODO: remove this after merging to main - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install dependencies' - - - script: | - - # Install necessary packages - sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake - sudo tdnf install binutils -y - sudo tdnf install libsecret -y + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin - export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # Verify gcc and make - gcc --version - make --version + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - df -h + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' - # Download and extract libsecret - wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - tar -xf libsecret-0.20.4.tar.xz - cd libsecret-0.20.4 + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - echo $PATH - echo $LD_LIBRARY_PATH + - script: | + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - # Configure and compile libsecret - ./configure - if [ $? -ne 0 ]; then - echo "Configuration failed. Check config.log for details." - cat config.log + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" exit 1 fi + displayName: 'Verify and extract the generated build version' - make - if [ $? -ne 0 ]; then - echo "Make failed." - exit 1 - fi + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - sudo make install - if [ $? -ne 0 ]; then - echo "Make install failed." - exit 1 - fi + linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - # Verify installation - ldconfig -p | grep libsecret + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + displayName: 'Create required directories' - displayName: 'Install libsecret' - - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # # Run azcopy --version and capture the output - # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # - script: | - # linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - # linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - # mkdir -p $linux_amd64_dir - # mkdir -p $linux_se_amd64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - # cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64_dir)/ - # cp NOTICE.txt $(linux_se_amd64_dir)/ - # displayName: 'Copy required files for packaging' - - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + displayName: 'Copy required files for packaging' + + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + - script: | + export GOARCH=arm64 + export GOOS=linux + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # - script: | - # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - # linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - # mkdir -p $linux_arm64_dir - # mkdir -p $linux_se_arm64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_arm64_dir)/ - # cp NOTICE.txt $(linux_se_arm64_dir)/ - # displayName: 'Copy required files for packaging' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - # displayName: 'Remove arm7 binary from staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy NOTICE file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - # - stage: SignArtifacts - # dependsOn: BuildArtifacts - # condition: succeeded('BuildArtifacts') - # jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: '*.rpm, *.deb, *tar.gz' - # DisplayName: 'ESRP CodeSigning azCopy linux' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/mariner - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - # Pattern: '*.rpm, *.deb' - # DisplayName: 'ESRP CodeSigning azCopy mariner' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - - # - script: | - # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir $windows_amd64 - # mkdir $windows_arm64 - # mkdir $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir $windows_amd64 + mkdir $windows_arm64 + mkdir $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy signed mac binary to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' - - # # SignArtifacts end here - - # - stage: TestArtifacts - # dependsOn: SignArtifacts - # condition: succeeded('SignArtifacts') - # jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt-get install libsecret-1-dev -y - # sudo apt --fix-broken install -y - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' + + # SignArtifacts end here + + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt-get install libsecret-1-dev -y + sudo apt --fix-broken install -y + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install libsecret -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install libsecret -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install libsecret -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install libsecret -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install dependencies' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' - # - script: | - # sudo tdnf install -y binutils gcc gcc-c++ make - # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make - # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - # tar -xf libsecret-0.20.4.tar.xz - # cd libsecret-0.20.4 - # ./configure - # make - # sudo make install - # ldconfig -p | grep libsecret - # displayName: 'Install libsecret' - - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo tdnf install gcc build-essential -y - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Find the directory matching the pattern - # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # # Run the azcopy version command - # ./azcopy --version - # displayName: 'Install Zip, Extract Files, and Run Version Command' + - script: | + sudo tdnf install -y binutils gcc gcc-c++ make + sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make + wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + tar -xf libsecret-0.20.4.tar.xz + cd libsecret-0.20.4 + ./configure + make + sudo make install + ldconfig -p | grep libsecret + displayName: 'Install libsecret' + + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo tdnf install gcc build-essential -y + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + displayName: 'Install Zip, Extract Files, and Run Version Command' - # - job: Set_7 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # # Find the zip file matching the pattern - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } - - # $zipFilePath = $zipFile.FullName - - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } - - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir - - # # Find directories matching the pattern - # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # # Check if there is exactly one matching directory - # if ($matchingDirs.Count -eq 0) { - # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } elseif ($matchingDirs.Count -gt 1) { - # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } - - # # Check if the azcopy executable exists in the matching directory - # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" - # if (-Not (Test-Path -Path $azcopyPath)) { - # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - # exit 1 - # } else { - # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # } - # & $azcopyPath --version - # displayName: 'Extract Files and Run Version Command on Windows' - - # # TestArtifacts ends here + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + } + & $azcopyPath --version + displayName: 'Extract Files and Run Version Command on Windows' + - job: Set_8 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + fuselib: 'libfuse3-dev' + fuse-version: 'fuse3' + tags: 'fuse3' + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install Git' + + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(root_dir)/azure-storage-azcopy + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "GoTool Custom Setup" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo tdnf install gcc build-essential fuse3 fuse3-devel -y + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + # TestArtifacts ends here - # - stage: ReleaseArtifacts - # dependsOn: TestArtifacts - # condition: succeeded('TestArtifacts') - # jobs: - # - job: ReleaseToContainer - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToContainer + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) - # variables: - # - name: Test_Release_Account_Name - # value: 'azcopyprivatedrops' - # - name: Release_Account_Name - # value: 'azcopyvnextrelease' - # - name: Test_Mode - # value: ${{ parameters.test_mode }} - - # steps: - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Linux Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Windows Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Mac Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + - name: Test_Mode + value: ${{ parameters.test_mode }} + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Binaries Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # cp NOTICE.txt azCopy-binaries-signed/ - # sudo apt-get install -y zip - # zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - # cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Create drop.zip file and install azcopy deb package' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # - task: ArchiveFiles@2 - # displayName: 'Archive all binaries to drop.zip' - # inputs: - # rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/*' - # archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' - # continueOnError: true - - # - task: Bash@3 - # displayName: 'Install Azure CLI' - # inputs: - # targetType: 'inline' - # script: | - # curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - # - task: AzureCLI@2 - # displayName: 'Azure CLI' - # inputs: - # azureSubscription: 'ESRP KeyVault identity' - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # # Find the current version - # raw_version_output=$(azcopy --version) - # azcopy_version=$(echo $raw_version_output | awk '{print $3}') - - # ls -ltR $(Build.ArtifactStagingDirectory) - - # today=$(date +"%Y%m%d") + - task: DownloadBuildArtifacts@0 + displayName: 'Download Binaries Build Artifacts' + inputs: + artifactName: 'azCopy-binaries-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cp NOTICE.txt azCopy-binaries-signed/ + sudo apt-get install -y zip + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Create drop.zip file and install azcopy deb package' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - task: ArchiveFiles@2 + displayName: 'Archive all binaries to drop.zip' + inputs: + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/*' + archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' + continueOnError: true + + - task: Bash@3 + displayName: 'Install Azure CLI' + inputs: + targetType: 'inline' + script: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=$(azcopy --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + ls -ltR $(Build.ArtifactStagingDirectory) + + today=$(date +"%Y%m%d") - # if [ $(Test_Mode) = "True" ]; then - # container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - # else - # container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - # fi - # echo "Artifacts will be uploaded to: $container_url" + if [ $(Test_Mode) = "True" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" - # executable_name=azcopy + executable_name=azcopy - # # Upload the archived builds - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - - # - job: ReleaseAzcopy - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Linux Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Windows Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Mac Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip - # displayName: 'List Artifacts' - - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # - ${{ if eq(parameters.post_release, true) }}: + - ${{ if eq(parameters.post_release, true) }}: - # # add release tags & push to github - # #TODO: update github connection and repo name while merging to main - # - task: GithubRelease@1 - # inputs: - # githubConnection: 'dphulkar-msft-github-rel' - # repositoryName: 'dphulkar-msft/azure-storage-azcopy' - # action: 'edit' + # add release tags & push to github + #TODO: update github connection and repo name while merging to main + - task: GithubRelease@1 + inputs: + githubConnection: 'dphulkar-msft-github-rel' + repositoryName: 'dphulkar-msft/azure-storage-azcopy' + action: 'edit' - # target: '$(Build.SourceVersion)' - # tagSource: 'userSpecifiedTag' + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' - # title: ${{ parameters.tag }} - # tag: ${{ parameters.tag }} + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} - # assets: | - # $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* - # $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* - # $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* - # changeLogCompareToRelease: 'lastFullRelease' - # changeLogType: 'commitBased' - # isDraft: ${{ parameters.draft }} - # isPreRelease: ${{ parameters.prerelease }} - # assetUploadMode: replace - - # - ${{ if eq(parameters.publish_artifacts, true) }}: - # - stage: PublishArtifacts - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: PublishArtifacts - # timeoutInMinutes: 120 - # pool: - # vmImage: 'ubuntu-22.04' - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + assets: | + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'DevExGlobalFeed' - # displayName: 'Connect to PMC artifact' - - # - script: | - # pip install pmc-cli - # displayName: 'Install pmc-cli' - - # - task: DownloadSecureFile@1 - # name: pmcCertificate - # displayName: 'Download pmc pem file' - # inputs: - # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - # - task: DownloadSecureFile@1 - # name: settings - # displayName: 'Download settings.toml file' - # inputs: - # secureFile: 'settings.toml' - - # - script: | - # pmc --version - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - # if [ $? -ne 0 ]; then - # exit 1 - # fi - # displayName: 'Test PMC installation' - - # # download artifacts that need to be published - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Signed Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + # download artifacts that need to be published + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # cd mariner - # arm64file=$(ls azcopy*.arm64.rpm) - # amd64file=$(ls azcopy*.x86_64.rpm) - # marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" - # marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" - # mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - # mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - # ls -lRt - # mv azcopy*.rpm .. - # cd .. - # rm -r mariner/ - # displayName: 'Rename Mariner binaries' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # displayName: 'List Artifacts' - - # - script: | - # mv azCopy-linux-signed/*tar.gz . - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed - # displayName: 'Upload files' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # # replace org name while merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv - # cat ./packages.csv - # displayName: 'Fetch packages.csv' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ - - # - script: | - # # Find the Mariner RPM file and upload - # marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - # marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - # echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - - # azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - # echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - - # azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - # echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + - script: | + cd mariner + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + mv azCopy-linux-signed/*tar.gz . + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + displayName: 'Upload files' + workingDirectory: $(Build.ArtifactStagingDirectory) + + # replace org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + # Find the Mariner RPM file and upload + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" - # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - # echo "Azcopy ARM DEB ID: $azcopyArmDeb" - - # azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') - # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - # echo "Azcopy ARM RPM ID: $azcopyArmRpm" - - # # Find the Mariner RPM file and upload - # marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - # marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - # marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - # echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - - # if [[ $marinerAmdRpmFile == *"preview"* ]]; then - # echo "Package $marinerAmdRpmFile contains 'preview'." - # # while IFS=, read -r distro fuseArchType repoName releaseName; do - # # echo "Uploading packages for $distro" - # # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # # done < <(tail -n +5 ../packages.csv) - # else - # echo "Package $package_name1 does not contain 'preview'." - # # while IFS=, read -r distro fuseArchType repoName releaseName; do - # # echo "Uploading packages for $distro" - # # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # # done < <(tail -n +3 ../packages.csv | head -n -2) - # fi - - # displayName: 'Add uploaded packages to repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - # # - script: | - # # while IFS=, read -r distro fuseArchType repoName releaseName; do - # # if [[ $fuseArchType == *"Arm"* ]] - # # then - # # echo "Skipping for ARM type on $distro" - # # else - # # echo "Publishing for $distro" - # # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # # fi - # # done < <(tail -n +3 ../packages.csv) - # # displayName: 'Publish the repository' - # # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + echo "Package $marinerAmdRpmFile contains 'preview'." + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +5 ../packages.csv) + else + echo "Package $package_name1 does not contain 'preview'." + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv | head -n -2) + fi + + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + # - script: | + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv) + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - # - stage: UpdateLatestVersion - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: UpdateVersion - # pool: - # vmImage: 'ubuntu-20.04' - # variables: - # - group: AZCOPY_TESTS_VAR - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - stage: UpdateLatestVersion + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - ${{ if eq(parameters.update_version, true) }}: - # - script: | - # sudo apt-get install python3 -y - # python3 --version - # displayName: 'Installing Python' - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + steps: + - checkout: none + + - ${{ if eq(parameters.update_version, true) }}: + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # # install azcopy - # - script: | - # cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - # ls azcopy*.arm64.deb - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # azcopy --version - # displayName: 'Installing azcopy' + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' - # # TODO: change org name while merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py - # ls -l - # displayName: 'Getting Python script' - # workingDirectory: $(root_dir) - - # - script: | - # pip install azure-storage-blob - # python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" - # displayName: 'Updating version number' - # workingDirectory: $(root_dir) + # TODO: change org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) From 365340d7958d1d2ada8c408c4e46cf48a5679708 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 28 Jun 2024 20:46:47 +0530 Subject: [PATCH 333/513] test mariner installations --- release-pipelines.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a6fe25478..78a8f3776 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1485,23 +1485,6 @@ stages: sudo tdnf install git -y displayName: 'Install Git' - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(root_dir)/azure-storage-azcopy - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "GoTool Custom Setup" - # get glibc version with which build is done - script: | ldd --version From 174e4358a3dd23d0253779fea092430c8d5dedf6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 09:09:51 +0530 Subject: [PATCH 334/513] test mariner installations --- release-pipelines.yml | 854 +++++++++++++++++++++--------------------- 1 file changed, 425 insertions(+), 429 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 78a8f3776..ecd1007c9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -742,7 +742,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp displayName: 'Make Artifacts executable' - script: | @@ -757,317 +757,317 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + # - script: | + # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" + # mkdir $windows_amd64 + # mkdir $windows_arm64 + # mkdir $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' - mkdir $windows_amd64 - mkdir $windows_arm64 - mkdir $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' + # - script: | + # cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac binary to staging directory' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy signed mac binary to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1075,167 +1075,167 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - steps: - - checkout: none - - script: | - sudo apt update - sudo apt-get install libsecret-1-dev -y - sudo apt --fix-broken install -y - ldd --version - displayName: "GLIBC Version" + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt-get install libsecret-1-dev -y + # sudo apt --fix-broken install -y + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install libsecret -y - sudo yum install git -y - displayName: 'Install Git' + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install libsecret -y + # sudo yum install git -y + # displayName: 'Install Git' - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install libsecret -y - sudo yum install git -y - displayName: 'Install Git' + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install libsecret -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - job: Set_5 timeoutInMinutes: 120 @@ -1260,15 +1260,11 @@ stages: displayName: 'Install dependencies' - script: | - sudo tdnf install -y binutils gcc gcc-c++ make - sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make - wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - tar -xf libsecret-0.20.4.tar.xz - cd libsecret-0.20.4 - ./configure - make - sudo make install - ldconfig -p | grep libsecret + sudo tdnf install -y libsecret + ls /usr/lib | grep libsecret + ls /usr/local/lib | grep libsecret + sudo ldconfig + export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH displayName: 'Install libsecret' From 289135829dd9e3a37b687f6b86af115699e680bc Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 09:36:16 +0530 Subject: [PATCH 335/513] test mariner installations --- release-pipelines.yml | 140 ++++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 68 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ecd1007c9..b4e739a6f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -993,81 +993,81 @@ stages: # displayName: 'Publish Signed Mac Binary' # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1260,11 +1260,15 @@ stages: displayName: 'Install dependencies' - script: | - sudo tdnf install -y libsecret - ls /usr/lib | grep libsecret - ls /usr/local/lib | grep libsecret + sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel + wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + tar -xf libsecret-0.20.4.tar.xz + cd libsecret-0.20.4 + ./configure + make + sudo make install sudo ldconfig - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + ldconfig -p | grep libsecret displayName: 'Install libsecret' From 3998806d1e6422d1da985a9a5c40c30e4758d8eb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 09:37:49 +0530 Subject: [PATCH 336/513] test mariner installations --- release-pipelines.yml | 138 +++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b4e739a6f..1c1154e11 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -993,81 +993,81 @@ stages: # displayName: 'Publish Signed Mac Binary' # - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here From 14da456748b7d095dfa5424c4bd26c43d43981e2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 10:00:10 +0530 Subject: [PATCH 337/513] test mariner installations --- release-pipelines.yml | 466 +++++++++++++++++++++--------------------- 1 file changed, 236 insertions(+), 230 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1c1154e11..9ad729824 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -302,271 +302,271 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' - - script: | - sudo ls -lRt $(root_dir)/ - cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + # - script: | + # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' + # linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - displayName: 'Copy required files for packaging' + # mkdir -p $linux_arm64_dir + # mkdir -p $linux_se_arm64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_arm64_dir)/ + # cp NOTICE.txt $(linux_se_arm64_dir)/ + # displayName: 'Copy required files for packaging' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - displayName: 'Remove arm7 binary from staging directory' + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + # displayName: 'Remove arm7 binary from staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_4 + # - job: Set_4 timeoutInMinutes: 360 strategy: matrix: @@ -1260,7 +1260,12 @@ stages: displayName: 'Install dependencies' - script: | - sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel + sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel + gcc --version + export CC=gcc + export CXX=g++ + df -h + sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz tar -xf libsecret-0.20.4.tar.xz cd libsecret-0.20.4 @@ -1269,6 +1274,7 @@ stages: sudo make install sudo ldconfig ldconfig -p | grep libsecret + cat config.log displayName: 'Install libsecret' From 6548e6a21ac17fcdc9c8d0123fe3eff0390bee54 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 10:01:10 +0530 Subject: [PATCH 338/513] test mariner installations --- release-pipelines.yml | 188 +++++++++++++++++++++--------------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9ad729824..851fa8bbe 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -566,103 +566,103 @@ stages: # verbosity: 'Verbose' # alertWarningLevel: 'High' - # - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy NOTICE file' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy NOTICE file' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts From f131911b6a1be3087d1fda9cc2e0520652f3f00a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 11:59:31 +0530 Subject: [PATCH 339/513] test mariner installations --- release-pipelines.yml | 961 ++++++++++++++++++++---------------------- 1 file changed, 454 insertions(+), 507 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 851fa8bbe..4b58c3b36 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -233,12 +233,13 @@ stages: - script: | mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 pkgDir/usr/bin/azcopy cp NOTICE.txt $(linux_amd64_dir)/ cp NOTICE.txt $(linux_se_amd64_dir)/ + cp NOTICE.txt pkgDir/usr/bin/ + ./pkgDir/usr/bin/azcopy --version displayName: 'Copy required files for packaging' - script: | @@ -302,269 +303,269 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + - script: | + export GOARCH=arm64 + export GOOS=linux + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # - script: | - # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - # linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - # mkdir -p $linux_arm64_dir - # mkdir -p $linux_se_arm64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_arm64_dir)/ - # cp NOTICE.txt $(linux_se_arm64_dir)/ - # displayName: 'Copy required files for packaging' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - # displayName: 'Remove arm7 binary from staging directory' + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + # displayName: 'Remove arm7 binary from staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # - job: Set_4 # timeoutInMinutes: 360 @@ -993,81 +994,81 @@ stages: # displayName: 'Publish Signed Mac Binary' # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1259,30 +1260,30 @@ stages: sudo tdnf install git -y displayName: 'Install dependencies' - - script: | - sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel - gcc --version - export CC=gcc - export CXX=g++ - df -h - sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel - wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - tar -xf libsecret-0.20.4.tar.xz - cd libsecret-0.20.4 - ./configure - make - sudo make install - sudo ldconfig - ldconfig -p | grep libsecret - cat config.log - displayName: 'Install libsecret' + # - script: | + # sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel + # gcc --version + # export CC=gcc + # export CXX=g++ + # df -h + # sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel + # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + # tar -xf libsecret-0.20.4.tar.xz + # cd libsecret-0.20.4 + # ./configure + # make + # sudo make install + # sudo ldconfig + # ldconfig -p | grep libsecret + # cat config.log + # displayName: 'Install libsecret' - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' # get glibc version with which build is done - script: | @@ -1291,230 +1292,176 @@ stages: - script: | sudo rpm -qip azcopy*x86_64.rpm - sudo tdnf install gcc build-essential -y + sudo tdnf groupinstall "Development Tools" -y sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/ - script: | azcopy --version azcopy --help displayName: 'Check Version and Help' - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Create extraction directory - mkdir -p $EXTRACT_DIR + # # Create extraction directory + # mkdir -p $EXTRACT_DIR - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + # # Find the directory matching the pattern + # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi - # Run the azcopy version command - ./azcopy --version - displayName: 'Install Zip, Extract Files, and Run Version Command' + # # Run the azcopy version command + # ./azcopy --version + # displayName: 'Install Zip, Extract Files, and Run Version Command' - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_7 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } + # # Find the zip file matching the pattern + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - # Change to the directory containing azcopy - Set-Location -Path $extractDir + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } - # Find directories matching the pattern - $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + # $zipFilePath = $zipFile.FullName - # Check if there is exactly one matching directory - if ($matchingDirs.Count -eq 0) { - Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } elseif ($matchingDirs.Count -gt 1) { - Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" - if (-Not (Test-Path -Path $azcopyPath)) { - Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - exit 1 - } else { - Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - } - & $azcopyPath --version - displayName: 'Extract Files and Run Version Command on Windows' - - job: Set_8 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - fuselib: 'libfuse3-dev' - fuse-version: 'fuse3' - tags: 'fuse3' - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install Git' + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + # # Find directories matching the pattern + # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo tdnf install gcc build-essential fuse3 fuse3-devel -y - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + # # Check if there is exactly one matching directory + # if ($matchingDirs.Count -eq 0) { + # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } elseif ($matchingDirs.Count -gt 1) { + # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - # TestArtifacts ends here + # # Check if the azcopy executable exists in the matching directory + # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + # if (-Not (Test-Path -Path $azcopyPath)) { + # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + # exit 1 + # } else { + # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # } + # & $azcopyPath --version + # displayName: 'Extract Files and Run Version Command on Windows' + + # # TestArtifacts ends here - stage: ReleaseArtifacts dependsOn: TestArtifacts From 6f787f23bd41e5c0b614c762d0bd25f9278504cb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 12:00:01 +0530 Subject: [PATCH 340/513] test mariner installations --- release-pipelines.yml | 138 +++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4b58c3b36..c5b838212 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -994,81 +994,81 @@ stages: # displayName: 'Publish Signed Mac Binary' # - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here From 09d9eca91ca931dc88360b00cedd9857872b841e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 12:10:59 +0530 Subject: [PATCH 341/513] test mariner installations --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c5b838212..cd05c7144 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -235,11 +235,12 @@ stages: mkdir -p pkgDir/usr/bin/ mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_amd64 pkgDir/usr/bin/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ cp NOTICE.txt $(linux_amd64_dir)/ cp NOTICE.txt $(linux_se_amd64_dir)/ cp NOTICE.txt pkgDir/usr/bin/ ./pkgDir/usr/bin/azcopy --version + ldd $(linux_amd64_dir)/azcopy displayName: 'Copy required files for packaging' - script: | From e16a7f4526d0ab9277005f1cd72e12466c02dfab Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 12:33:26 +0530 Subject: [PATCH 342/513] test libsecret installation on mariner --- release-pipelines.yml | 763 +++++++++++++++++++++--------------------- 1 file changed, 386 insertions(+), 377 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index cd05c7144..edf97a6bf 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,447 +44,456 @@ pr: none stages: - stage: BuildArtifacts jobs: - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) - # steps: - # - checkout: none - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install dependencies' + steps: + - checkout: none + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' - # - script: | + - script: | + # Install libsecret if not already installed + sudo tdnf install -y libsecret - # # Install necessary packages - # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake - # sudo tdnf install binutils -y - # sudo tdnf install libsecret -y + # List all files installed by libsecret package + tdnf repoquery -l libsecret - # export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin - # export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib + # Search for libsecret library specifically + find /usr -name 'libsecret-1.so*' - # # Verify gcc and make - # gcc --version - # make --version - # df -h + # # Install necessary packages + # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake + # sudo tdnf install binutils -y + # sudo tdnf install libsecret -y - # # Download and extract libsecret - # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - # tar -xf libsecret-0.20.4.tar.xz - # cd libsecret-0.20.4 + # export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + # export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib - # echo $PATH - # echo $LD_LIBRARY_PATH + # # Verify gcc and make + # gcc --version + # make --version - # # Configure and compile libsecret - # ./configure - # if [ $? -ne 0 ]; then - # echo "Configuration failed. Check config.log for details." - # cat config.log - # exit 1 - # fi + # df -h - # make - # if [ $? -ne 0 ]; then - # echo "Make failed." - # exit 1 - # fi + # # Download and extract libsecret + # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + # tar -xf libsecret-0.20.4.tar.xz + # cd libsecret-0.20.4 - # sudo make install - # if [ $? -ne 0 ]; then - # echo "Make install failed." - # exit 1 - # fi + # echo $PATH + # echo $LD_LIBRARY_PATH + + # # Configure and compile libsecret + # ./configure + # if [ $? -ne 0 ]; then + # echo "Configuration failed. Check config.log for details." + # cat config.log + # exit 1 + # fi - # # Verify installation - # ldconfig -p | grep libsecret + # make + # if [ $? -ne 0 ]; then + # echo "Make failed." + # exit 1 + # fi - # displayName: 'Install libsecret' + # sudo make install + # if [ $? -ne 0 ]; then + # echo "Make install failed." + # exit 1 + # fi + + # # Verify installation + # ldconfig -p | grep libsecret + + displayName: 'Install libsecret' - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' - - script: | - sudo ls -lRt $(root_dir)/ - cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # # Run azcopy --version and capture the output + # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + # - script: | + # linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + # linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - displayName: 'Create required directories' + # mkdir -p $linux_amd64_dir + # mkdir -p $linux_se_amd64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' - - script: | - mkdir -p pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ - cp NOTICE.txt pkgDir/usr/bin/ - ./pkgDir/usr/bin/azcopy --version - ldd $(linux_amd64_dir)/azcopy - displayName: 'Copy required files for packaging' + # - script: | + # mkdir -p pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + # cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64_dir)/ + # cp NOTICE.txt $(linux_se_amd64_dir)/ + # cp NOTICE.txt pkgDir/usr/bin/ + # ./pkgDir/usr/bin/azcopy --version + # ldd $(linux_amd64_dir)/azcopy + # displayName: 'Copy required files for packaging' - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' + # - script: | + # sudo ls -lRt pkgDir/usr/bin/ + # displayName: 'List the files' - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" - displayName: "Installing Go tools" + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/" + # displayName: "Installing Go tools" - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' - - script: | - sudo ls -lRt $(root_dir)/ - cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part using grep and sed - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # # Extract the version part using grep and sed + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + # - script: | + # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + # linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' + # mkdir -p $linux_arm64_dir + # mkdir -p $linux_se_arm64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_arm64_dir)/ + # cp NOTICE.txt $(linux_se_arm64_dir)/ + # displayName: 'Copy required files for packaging' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' # - job: Set_3 # timeoutInMinutes: 120 From 725c82c3a6065ae70aeb1fa3396dc6668d2c47fb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 12:44:31 +0530 Subject: [PATCH 343/513] test libsecret installation on mariner --- release-pipelines.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index edf97a6bf..b3c67e227 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -61,20 +61,16 @@ stages: steps: - checkout: none - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install dependencies' - script: | # Install libsecret if not already installed sudo tdnf install -y libsecret - + sudo rm /var/run/.tdnf-instance-lockfile # List all files installed by libsecret package - tdnf repoquery -l libsecret + sudo tdnf repoquery -l libsecret # Search for libsecret library specifically - find /usr -name 'libsecret-1.so*' + sudo find /usr -name 'libsecret-1.so*' # # Install necessary packages From 68f12b84ec5795f460ae9f99655b4eb3e446d32c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 13:10:22 +0530 Subject: [PATCH 344/513] test ls --- release-pipelines.yml | 469 +++++++++++++++++++++--------------------- 1 file changed, 235 insertions(+), 234 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b3c67e227..65972f444 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,270 +44,271 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - script: | - # Install libsecret if not already installed - sudo tdnf install -y libsecret - sudo rm /var/run/.tdnf-instance-lockfile - # List all files installed by libsecret package - sudo tdnf repoquery -l libsecret - - # Search for libsecret library specifically - sudo find /usr -name 'libsecret-1.so*' - - - # # Install necessary packages - # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake - # sudo tdnf install binutils -y - # sudo tdnf install libsecret -y - - # export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin - # export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib - - # # Verify gcc and make - # gcc --version - # make --version - - # df -h - - # # Download and extract libsecret - # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - # tar -xf libsecret-0.20.4.tar.xz - # cd libsecret-0.20.4 - - # echo $PATH - # echo $LD_LIBRARY_PATH - - # # Configure and compile libsecret - # ./configure - # if [ $? -ne 0 ]; then - # echo "Configuration failed. Check config.log for details." - # cat config.log - # exit 1 - # fi - - # make - # if [ $? -ne 0 ]; then - # echo "Make failed." - # exit 1 - # fi - - # sudo make install - # if [ $? -ne 0 ]; then - # echo "Make install failed." - # exit 1 - # fi - - # # Verify installation - # ldconfig -p | grep libsecret - - displayName: 'Install libsecret' - - # - job: Set_1 + # - job: Set_5 # timeoutInMinutes: 120 # strategy: # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + # pool: - # name: "blobfuse-ubuntu-pool" + # name: "blobfuse-mariner-pool" # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives + # - ImageOverride -equals $(AgentName) # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + # - checkout: none + + # - script: | + # # Install libsecret if not already installed + # sudo tdnf install -y libsecret + # sudo rm /var/run/.tdnf-instance-lockfile + # # List all files installed by libsecret package + # sudo tdnf repoquery -l libsecret + + # # Search for libsecret library specifically + # sudo find /usr -name 'libsecret-1.so*' + + + # # # Install necessary packages + # # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake + # # sudo tdnf install binutils -y + # # sudo tdnf install libsecret -y + + # # export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + # # export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib + + # # # Verify gcc and make + # # gcc --version + # # make --version + + # # df -h + + # # # Download and extract libsecret + # # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz + # # tar -xf libsecret-0.20.4.tar.xz + # # cd libsecret-0.20.4 + + # # echo $PATH + # # echo $LD_LIBRARY_PATH + + # # # Configure and compile libsecret + # # ./configure + # # if [ $? -ne 0 ]; then + # # echo "Configuration failed. Check config.log for details." + # # cat config.log + # # exit 1 + # # fi + + # # make + # # if [ $? -ne 0 ]; then + # # echo "Make failed." + # # exit 1 + # # fi + + # # sudo make install + # # if [ $? -ne 0 ]; then + # # echo "Make install failed." + # # exit 1 + # # fi + + # # # Verify installation + # # ldconfig -p | grep libsecret + + # displayName: 'Install libsecret' + + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # # Run azcopy --version and capture the output - # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + - script: | + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # - script: | - # linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - # linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - # mkdir -p $linux_amd64_dir - # mkdir -p $linux_se_amd64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + ls -ltR $(work_dir) + displayName: 'Create required directories' - # - script: | - # mkdir -p pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - # cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64_dir)/ - # cp NOTICE.txt $(linux_se_amd64_dir)/ - # cp NOTICE.txt pkgDir/usr/bin/ - # ./pkgDir/usr/bin/azcopy --version - # ldd $(linux_amd64_dir)/azcopy - # displayName: 'Copy required files for packaging' + - script: | + mkdir -p pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + cp NOTICE.txt pkgDir/usr/bin/ + ./pkgDir/usr/bin/azcopy --version + ldd $(linux_amd64_dir)/azcopy + displayName: 'Copy required files for packaging' - # - script: | - # sudo ls -lRt pkgDir/usr/bin/ - # displayName: 'List the files' + - script: | + sudo ls -lRt pkgDir/usr/bin/ + displayName: 'List the files' - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # - job: Set_2 # timeoutInMinutes: 120 From aa22106bdb0aaa8914fd53b2f73d405111527b7e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 13:44:05 +0530 Subject: [PATCH 345/513] test mariner --- release-pipelines.yml | 162 ++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 91 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 65972f444..ae0cbbbd6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -228,7 +228,7 @@ stages: linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - linux_se_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" mkdir -p $linux_amd64_dir @@ -1001,81 +1001,81 @@ stages: # displayName: 'Publish Signed Mac Binary' # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1266,26 +1266,7 @@ stages: sudo tdnf update -y sudo tdnf install git -y displayName: 'Install dependencies' - - # - script: | - # sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel - # gcc --version - # export CC=gcc - # export CXX=g++ - # df -h - # sudo tdnf install -y binutils gcc gcc-c++ make glib2-devel libffi-devel - # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - # tar -xf libsecret-0.20.4.tar.xz - # cd libsecret-0.20.4 - # ./configure - # make - # sudo make install - # sudo ldconfig - # ldconfig -p | grep libsecret - # cat config.log - # displayName: 'Install libsecret' - - + - template: setup/test_artifacts.yml parameters: artifact_name: 'azCopy-linux-signed' @@ -1299,7 +1280,6 @@ stages: - script: | sudo rpm -qip azcopy*x86_64.rpm - sudo tdnf groupinstall "Development Tools" -y sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/ From f8301d1409eb3cf854ca9ee0c6ef7be69cd7dc1f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 13:47:59 +0530 Subject: [PATCH 346/513] test mariner --- release-pipelines.yml | 500 ++++++++++++++++++------------------------ 1 file changed, 212 insertions(+), 288 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ae0cbbbd6..03244023e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,82 +44,6 @@ pr: none stages: - stage: BuildArtifacts jobs: - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - - # - script: | - # # Install libsecret if not already installed - # sudo tdnf install -y libsecret - # sudo rm /var/run/.tdnf-instance-lockfile - # # List all files installed by libsecret package - # sudo tdnf repoquery -l libsecret - - # # Search for libsecret library specifically - # sudo find /usr -name 'libsecret-1.so*' - - - # # # Install necessary packages - # # sudo tdnf install -y gcc glib2-devel gnome-keyring-devel make libtool autoconf automake - # # sudo tdnf install binutils -y - # # sudo tdnf install libsecret -y - - # # export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin - # # export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib - - # # # Verify gcc and make - # # gcc --version - # # make --version - - # # df -h - - # # # Download and extract libsecret - # # wget https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz - # # tar -xf libsecret-0.20.4.tar.xz - # # cd libsecret-0.20.4 - - # # echo $PATH - # # echo $LD_LIBRARY_PATH - - # # # Configure and compile libsecret - # # ./configure - # # if [ $? -ne 0 ]; then - # # echo "Configuration failed. Check config.log for details." - # # cat config.log - # # exit 1 - # # fi - - # # make - # # if [ $? -ne 0 ]; then - # # echo "Make failed." - # # exit 1 - # # fi - - # # sudo make install - # # if [ $? -ne 0 ]; then - # # echo "Make install failed." - # # exit 1 - # # fi - - # # # Verify installation - # # ldconfig -p | grep libsecret - - # displayName: 'Install libsecret' - - job: Set_1 timeoutInMinutes: 120 strategy: @@ -310,187 +234,187 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/" - # displayName: "Installing Go tools" + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/" + displayName: "Installing Go tools" - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # # Extract the version part using grep and sed - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + - script: | + export GOARCH=arm64 + export GOOS=linux + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # Extract the version part using grep and sed + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' - # - script: | - # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - # linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - # mkdir -p $linux_arm64_dir - # mkdir -p $linux_se_arm64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_arm64_dir)/ - # cp NOTICE.txt $(linux_se_arm64_dir)/ - # displayName: 'Copy required files for packaging' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' # - job: Set_3 # timeoutInMinutes: 120 @@ -1016,66 +940,66 @@ stages: # - group: AZCOPY_ESRP_SECRET # steps: - - checkout: none + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here From 190efb004af9f9d7aa26b4438f0be6c250227bd9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 14:25:46 +0530 Subject: [PATCH 347/513] complete test --- go_installer.sh | 2 +- release-pipelines.yml | 1601 +++++++++++++++++++------------------- setup/test_artifacts.yml | 5 +- 3 files changed, 806 insertions(+), 802 deletions(-) diff --git a/go_installer.sh b/go_installer.sh index 932951471..d1f569fd1 100644 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.21.8" +version=$2 arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ];then diff --git a/release-pipelines.yml b/release-pipelines.yml index 03244023e..cf8a73e48 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -77,7 +77,7 @@ stages: - task: ShellScript@2 inputs: scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" displayName: "Installing Go tools" - script: | @@ -268,7 +268,7 @@ stages: - task: ShellScript@2 inputs: scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" displayName: "Installing Go tools" - script: | @@ -416,185 +416,185 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm - # displayName: 'Remove arm7 binary from staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy NOTICE file' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy NOTICE file' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -689,317 +689,317 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - - # - script: | - # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir $windows_amd64 - # mkdir $windows_arm64 - # mkdir $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir $windows_amd64 + mkdir $windows_arm64 + mkdir $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - # cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy signed mac binary to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1007,167 +1007,168 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt-get install libsecret-1-dev -y - # sudo apt --fix-broken install -y - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install -y + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install libsecret -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install libsecret -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_5 timeoutInMinutes: 120 @@ -1193,9 +1194,10 @@ stages: - template: setup/test_artifacts.yml parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' # get glibc version with which build is done - script: | @@ -1206,173 +1208,173 @@ stages: sudo rpm -qip azcopy*x86_64.rpm sudo rpm -i azcopy*x86_64.rpm displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/ + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - script: | azcopy --version azcopy --help displayName: 'Check Version and Help' - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Find the directory matching the pattern - # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # # Run the azcopy version command - # ./azcopy --version - # displayName: 'Install Zip, Extract Files, and Run Version Command' + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + displayName: 'Install Zip, Extract Files, and Run Version Command' - # - job: Set_7 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # # Find the zip file matching the pattern - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } - - # $zipFilePath = $zipFile.FullName - - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } - - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir - - # # Find directories matching the pattern - # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # # Check if there is exactly one matching directory - # if ($matchingDirs.Count -eq 0) { - # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } elseif ($matchingDirs.Count -gt 1) { - # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } - - # # Check if the azcopy executable exists in the matching directory - # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" - # if (-Not (Test-Path -Path $azcopyPath)) { - # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - # exit 1 - # } else { - # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # } - # & $azcopyPath --version - # displayName: 'Extract Files and Run Version Command on Windows' + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + } + & $azcopyPath --version + displayName: 'Extract Files and Run Version Command on Windows' - # # TestArtifacts ends here + # TestArtifacts ends here - stage: ReleaseArtifacts dependsOn: TestArtifacts @@ -1674,10 +1676,10 @@ stages: if [[ $marinerAmdRpmFile == *"preview"* ]]; then echo "Package $marinerAmdRpmFile contains 'preview'." - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +5 ../packages.csv) + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +27 ../packages.csv) else echo "Package $package_name1 does not contain 'preview'." # while IFS=, read -r distro fuseArchType repoName releaseName; do @@ -1685,22 +1687,21 @@ stages: # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName # done < <(tail -n +3 ../packages.csv | head -n -2) fi - displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - # - script: | - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +3 ../packages.csv) - # displayName: 'Publish the repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + - script: | + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +27 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion dependsOn: ReleaseArtifacts diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index f09b47044..70396f0ed 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -1,4 +1,6 @@ parameters: + - name: go_version + type: string - name: artifact_name type: string - name: download_path @@ -7,6 +9,7 @@ parameters: type: string steps: + #TODO: Change the org name after merging to main - script: | git clone https://github.com/dphulkar-msft/azure-storage-azcopy displayName: 'Checkout Code' @@ -20,7 +23,7 @@ steps: - task: ShellScript@2 inputs: scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" - args: "$(System.DefaultWorkingDirectory)/" + args: "$(System.DefaultWorkingDirectory)/ ${{ parameters.go_version }}" displayName: "GoTool Custom Setup" - task: DownloadBuildArtifacts@0 From e0dc91b542b09ecc614ac5f7345eb49461e7fc40 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 15:02:57 +0530 Subject: [PATCH 348/513] complete test --- release-pipelines.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index cf8a73e48..8007f7f3f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -169,8 +169,6 @@ stages: cp NOTICE.txt $(linux_amd64_dir)/ cp NOTICE.txt $(linux_se_amd64_dir)/ cp NOTICE.txt pkgDir/usr/bin/ - ./pkgDir/usr/bin/azcopy --version - ldd $(linux_amd64_dir)/azcopy displayName: 'Copy required files for packaging' - script: | @@ -341,7 +339,7 @@ stages: linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - linux_se_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" mkdir -p $linux_arm64_dir From dc76ab7ffabb2a365bc574926ccf6fb52775d043 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 16:30:12 +0530 Subject: [PATCH 349/513] complete test --- release-pipelines.yml | 77 ++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8007f7f3f..5c4010c72 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -130,7 +130,7 @@ stages: # Run azcopy --version and capture the output azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - # Extract the version part using grep and sed + # Extract the version part if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') if [[ -n "$version" ]]; then @@ -171,10 +171,6 @@ stages: cp NOTICE.txt pkgDir/usr/bin/ displayName: 'Copy required files for packaging' - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | @@ -312,12 +308,13 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" + - script: | export GOARCH=arm64 export GOOS=linux azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - # Extract the version part using grep and sed + # Extract the version part if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') if [[ -n "$version" ]]; then @@ -560,7 +557,7 @@ stages: cp NOTICE.txt $(darwin_arm_dir) mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy NOTICE file' + displayName: 'Copy required files for packaging' - task: ArchiveFiles@2 displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' @@ -577,8 +574,7 @@ stages: continueOnError: true - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 @@ -777,9 +773,9 @@ stages: windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" echo "##vso[task.setvariable variable=windows_386]$windows_386" - mkdir $windows_amd64 - mkdir $windows_arm64 - mkdir $windows_386 + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 mkdir -p $(archives) displayName: 'Create required directories' @@ -817,9 +813,7 @@ stages: continueOnError: true - script: | - cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - script: | @@ -914,7 +908,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac binary to staging directory' + displayName: 'Copy and list signed mac binary to staging directory' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 @@ -1120,7 +1114,7 @@ stages: - script: | azcopy --version azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help command' - job: Set_4 timeoutInMinutes: 120 @@ -1166,7 +1160,7 @@ stages: - script: | azcopy --version azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help command' - job: Set_5 timeoutInMinutes: 120 @@ -1185,6 +1179,7 @@ stages: steps: - checkout: none + - script: | sudo tdnf update -y sudo tdnf install git -y @@ -1211,7 +1206,7 @@ stages: - script: | azcopy --version azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help command' - job: Set_6 timeoutInMinutes: 360 @@ -1281,7 +1276,8 @@ stages: # Run the azcopy version command ./azcopy --version - displayName: 'Install Zip, Extract Files, and Run Version Command' + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - job: Set_7 timeoutInMinutes: 120 @@ -1370,7 +1366,8 @@ stages: Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" } & $azcopyPath --version - displayName: 'Extract Files and Run Version Command on Windows' + & $azcopyPath --help + displayName: 'Extract Files and Run Version and Help Command on Windows' # TestArtifacts ends here @@ -1425,7 +1422,6 @@ stages: - script: | cp NOTICE.txt azCopy-binaries-signed/ sudo apt-get install -y zip - zip -r $(Build.ArtifactStagingDirectory)/drop.zip . cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb @@ -1433,12 +1429,18 @@ stages: displayName: 'Create drop.zip file and install azcopy deb package' workingDirectory: $(Build.ArtifactStagingDirectory) - - task: ArchiveFiles@2 + #TODO: check once + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" + ls -l + + # Create the zip archive with all files inside azCopy-binaries-signed + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + + # List the contents of the resulting zip file for verification + unzip -l $(Build.ArtifactStagingDirectory)/drop.zip displayName: 'Archive all binaries to drop.zip' - inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed/*' - archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' - continueOnError: true - task: Bash@3 displayName: 'Install Azure CLI' @@ -1458,8 +1460,6 @@ stages: raw_version_output=$(azcopy --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') - ls -ltR $(Build.ArtifactStagingDirectory) - today=$(date +"%Y%m%d") if [ $(Test_Mode) = "True" ]; then @@ -1518,6 +1518,7 @@ stages: sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip displayName: 'List Artifacts' @@ -1604,7 +1605,6 @@ stages: fi displayName: 'Test PMC installation' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Signed Artifacts' inputs: @@ -1619,7 +1619,7 @@ stages: marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - ls -lRt + sudo ls -lRt mv azcopy*.rpm .. cd .. rm -r mariner/ @@ -1636,7 +1636,7 @@ stages: displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) - # replace org name while merging to main + # TODO: replace org name while merging to main - script: | wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv cat ./packages.csv @@ -1644,32 +1644,33 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - # Find the Mariner RPM file and upload - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB $azcopyAmdDeb" echo "Azcopy AMD DEB ID: $azcopyAmdDeb" azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + echo "Azcopy AMD RPM $azcopyAmdRpm" + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB $azcopyArmDeb" echo "Azcopy ARM DEB ID: $azcopyArmDeb" azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" echo "Azcopy ARM RPM ID: $azcopyArmRpm" # Find the Mariner RPM file and upload marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM: $marinerArmRpmFile" echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" if [[ $marinerAmdRpmFile == *"preview"* ]]; then From e6178662f0976d9a1f4ae5923eb54aaf0745c89b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 17:22:20 +0530 Subject: [PATCH 350/513] complete test --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5c4010c72..a21885050 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -903,8 +903,8 @@ stages: mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy-*/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy-*/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) @@ -1650,14 +1650,14 @@ stages: azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy AMD RPM $azcopyAmdRpm" + echo "Azcopy AMD RPM $azcopyAmdRpmFile" echo "Azcopy AMD RPM ID: $azcopyAmdRpm" azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` echo "Azcopy ARM DEB $azcopyArmDeb" echo "Azcopy ARM DEB ID: $azcopyArmDeb" - azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) echo "Azcopy ARM RPM $azcopyArmRpmFile" echo "Azcopy ARM RPM ID: $azcopyArmRpm" From 81ba88a480353047a2e22fc04fed0bafb9d0b814 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 17:46:02 +0530 Subject: [PATCH 351/513] complete test --- release-pipelines.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a21885050..44b7be3fc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1365,8 +1365,14 @@ stages: } else { Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" } - & $azcopyPath --version - & $azcopyPath --help + + # Run azcopy --version and capture output + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + + # Run azcopy --help and capture output + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" displayName: 'Extract Files and Run Version and Help Command on Windows' # TestArtifacts ends here @@ -1427,7 +1433,6 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: $(Build.ArtifactStagingDirectory) #TODO: check once - script: | From 9d09a36992eee45b2745a775164c39a1be9c9a25 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 17:46:02 +0530 Subject: [PATCH 352/513] complete test --- packages.csv | 4 +- release-pipelines.yml | 98 ++++++++++++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 26 deletions(-) diff --git a/packages.csv b/packages.csv index e7cd95d29..06dccaba8 100644 --- a/packages.csv +++ b/packages.csv @@ -1,7 +1,7 @@ # Do not remove these comments # Format of the file : ,,, Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, +Mariner-2.0-aarch64,marinerAarchRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, Ubuntu-18.04,azcopyAmdDeb,microsoft-ubuntu-bionic-prod-apt,bionic Ubuntu-20.04,azcopyAmdDeb,microsoft-ubuntu-focal-prod-apt,focal Ubuntu-22.04,azcopyAmdDeb,microsoft-ubuntu-jammy-prod-apt,jammy @@ -25,4 +25,4 @@ SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-preview-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-preview-Microsoft-aarch64-yum, \ No newline at end of file +Mariner-2.0-aarch64,marinerAarchRpm,cbl-mariner-2.0-preview-Microsoft-aarch64-yum, \ No newline at end of file diff --git a/release-pipelines.yml b/release-pipelines.yml index a21885050..93f236267 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -903,8 +903,24 @@ stages: mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy-*/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy-*/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") + echo "Folder name: $amd64FolderName" + + arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") + echo "Folder name: $arm64FolderName" + + # List the contents of the folder + echo "Contents:" + ls -l $amd64FolderName + ls -l $arm64FolderName + + # Attempt the mv command + mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) @@ -1364,9 +1380,25 @@ stages: exit 1 } else { Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } } - & $azcopyPath --version - & $azcopyPath --help displayName: 'Extract Files and Run Version and Help Command on Windows' # TestArtifacts ends here @@ -1427,7 +1459,6 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: $(Build.ArtifactStagingDirectory) #TODO: check once - script: | @@ -1668,37 +1699,56 @@ stages: echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM: $marinerArmRpmFile" - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" + marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + + is_preview="false" + echo "##vso[task.setvariable variable=is_preview]$is_preview" if [[ $marinerAmdRpmFile == *"preview"* ]]; then + is_preview="true" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + echo "Package $marinerAmdRpmFile contains 'preview'." while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +27 ../packages.csv) + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +5 ../packages.csv) else echo "Package $package_name1 does not contain 'preview'." - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +3 ../packages.csv | head -n -2) + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv | head -n -2) fi displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +27 ../packages.csv) + if [ "$(is_preview)" = "true" ]; then + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +5 ../packages.csv) + else + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv | head -n -2) + fi + displayName: 'Publish the repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ From 171b252d43ccb28778ac09584faa675608ff7f5b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 21:24:23 +0530 Subject: [PATCH 353/513] complete test --- setup/releaseVersionUpdate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index 95e1d4b27..c35de9443 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -17,9 +17,9 @@ sasToken = sasUrl.split('?')[1] # delete latest version file in the container -deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken -resp = requests.delete(deleteUrl) -sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') +# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +# resp = requests.delete(deleteUrl) +# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') # Create a file and write data to it file_name = 'latest_version.txt' From cd5193a6ff4033db0445c5ff6ff4905d8dfa7c3d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 1 Jul 2024 23:04:22 +0530 Subject: [PATCH 354/513] complete test --- packages.csv | 2 +- release-pipelines.yml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages.csv b/packages.csv index 06dccaba8..a6a984885 100644 --- a/packages.csv +++ b/packages.csv @@ -25,4 +25,4 @@ SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-preview-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerAarchRpm,cbl-mariner-2.0-preview-Microsoft-aarch64-yum, \ No newline at end of file +Mariner-2.0-aarch64,marinerAarchRpm,cbl-mariner-2.0-preview-Microsoft-aarch64-yum, diff --git a/release-pipelines.yml b/release-pipelines.yml index 93f236267..410a2baf2 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -439,7 +439,7 @@ stages: workingDirectory: $(root_dir) - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" displayName: 'Generate Windows AMD64' env: GOARCH: amd64 @@ -447,7 +447,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" displayName: 'Generate Windows i386' env: GOARCH: 386 @@ -455,7 +455,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" displayName: 'Generate Windows ARM' env: GOARCH: arm @@ -464,7 +464,7 @@ stages: CGO_ENABLED: 0 - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64" + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" displayName: 'Generate Windows ARM' env: GOARCH: arm64 @@ -478,7 +478,7 @@ stages: condition: succeededOrFailed() - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe displayName: 'Remove arm7 binary from staging directory' - task: PublishBuildArtifacts@1 @@ -780,9 +780,9 @@ stages: displayName: 'Create required directories' - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64 $(windows_amd64)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64 $(windows_arm64)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386 $(windows_386)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) @@ -1700,7 +1700,7 @@ stages: echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" From b49de49aa6c56a3ceb925371b06a2de89eaddde7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 09:50:19 +0530 Subject: [PATCH 355/513] windows complete test --- release-pipelines.yml | 1964 ++++++++++++++++++++--------------------- 1 file changed, 982 insertions(+), 982 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 410a2baf2..9ea363e1c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,372 +44,372 @@ pr: none stages: - stage: BuildArtifacts jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(root_dir)/ - cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + # displayName: 'Generate Linux AMD64' + + # - script: | + # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + # displayName: 'Generate Linux AMD64 SE Integration' + + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # export GOARCH=amd64 + # export GOOS=linux + # # Run azcopy --version and capture the output + # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # # Extract the version part + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - ls -ltR $(work_dir) - displayName: 'Create required directories' - - - script: | - mkdir -p pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ - cp NOTICE.txt pkgDir/usr/bin/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # - script: | + # linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + # linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + # mkdir -p $linux_amd64_dir + # mkdir -p $linux_se_amd64_dir + # mkdir -p $(archives) + # ls -ltR $(work_dir) + # displayName: 'Create required directories' + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + # cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_amd64_dir)/ + # cp NOTICE.txt $(linux_se_amd64_dir)/ + # cp NOTICE.txt pkgDir/usr/bin/ + # displayName: 'Copy required files for packaging' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make AMD64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_amd64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Amd Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_amd64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # #TODO: remove this after merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(root_dir)/ - cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" + + # - script: | + # export GOARCH=arm64 + # export GOOS=linux + # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # # Extract the version part + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' + + # - script: | + # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + # linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + # mkdir -p $linux_arm64_dir + # mkdir -p $linux_se_arm64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_arm64_dir)/ + # cp NOTICE.txt $(linux_se_arm64_dir)/ + # displayName: 'Copy required files for packaging' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to signing stage' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -493,195 +493,195 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - displayName: 'List Artifacts' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, *tar.gz' - DisplayName: 'ESRP CodeSigning azCopy linux' - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/mariner - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - Pattern: '*.rpm, *.deb' - DisplayName: 'ESRP CodeSigning azCopy mariner' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Linux-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # displayName: 'List Artifacts' + + # - script: | + # mkdir mariner && chmod 755 mariner + # cp azCopy-linux-temp/*arm64.rpm mariner + # cp azCopy-linux-temp/*x86_64.rpm mariner + # sudo ls -lRt mariner + # md5sum mariner/* + # displayName: 'Copy artifacts for Mariner' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: '*.rpm, *.deb, *tar.gz' + # DisplayName: 'ESRP CodeSigning azCopy linux' + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/mariner + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + # Pattern: '*.rpm, *.deb' + # DisplayName: 'ESRP CodeSigning azCopy mariner' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # md5sum $(Build.ArtifactStagingDirectory)/mariner/* + # displayName: 'List Signed Artifacts' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-signed' + # displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -828,186 +828,186 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") - echo "Folder name: $amd64FolderName" - - arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") - echo "Folder name: $arm64FolderName" + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") + # echo "Folder name: $amd64FolderName" + + # arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") + # echo "Folder name: $arm64FolderName" - # List the contents of the folder - echo "Contents:" - ls -l $amd64FolderName - ls -l $arm64FolderName - - # Attempt the mv command - mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # List the contents of the folder + # echo "Contents:" + # ls -l $amd64FolderName + # ls -l $arm64FolderName + + # # Attempt the mv command + # mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1015,285 +1015,285 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install -y - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install -y + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install dependencies' + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' + + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install dependencies' - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_6 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi - - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run the azcopy version command - ./azcopy --version - ./azcopy --help - displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo rpm -qip azcopy*x86_64.rpm + # sudo rpm -i azcopy*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' + + # - job: Set_6 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi + + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # # Create extraction directory + # mkdir -p $EXTRACT_DIR + + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # # Find the directory matching the pattern + # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi + + # # Run the azcopy version command + # ./azcopy --version + # ./azcopy --help + # displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - job: Set_7 timeoutInMinutes: 120 @@ -1374,7 +1374,7 @@ stages: } # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy" + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" if (-Not (Test-Path -Path $azcopyPath)) { Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" exit 1 From b0efb5f7b52ef795026f3ff8efed67ff79a5c2f0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 10:30:23 +0530 Subject: [PATCH 356/513] complete test --- release-pipelines.yml | 1946 ++++++++++++++++++++--------------------- 1 file changed, 973 insertions(+), 973 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9ea363e1c..58114b23c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -44,372 +44,372 @@ pr: none stages: - stage: BuildArtifacts jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - # displayName: 'Generate Linux AMD64' - - # - script: | - # GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - # displayName: 'Generate Linux AMD64 SE Integration' - - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # export GOARCH=amd64 - # export GOOS=linux - # # Run azcopy --version and capture the output - # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # # Extract the version part - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # - script: | - # linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - # linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - # mkdir -p $linux_amd64_dir - # mkdir -p $linux_se_amd64_dir - # mkdir -p $(archives) - # ls -ltR $(work_dir) - # displayName: 'Create required directories' - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - # cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_amd64_dir)/ - # cp NOTICE.txt $(linux_se_amd64_dir)/ - # cp NOTICE.txt pkgDir/usr/bin/ - # displayName: 'Copy required files for packaging' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + ls -ltR $(work_dir) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + cp NOTICE.txt pkgDir/usr/bin/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make AMD64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_amd64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Amd Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_amd64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" - - # - script: | - # export GOARCH=arm64 - # export GOOS=linux - # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # # Extract the version part - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + export GOARCH=arm64 + export GOOS=linux + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" - # # Set the pipeline variable (for Azure DevOps) - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # else - # echo "Unable to extract version from azcopy output" - # exit 1 - # fi - # else - # echo "azcopy version string not found in output" - # exit 1 - # fi - # displayName: 'Verify and extract the generated build version' - - # - script: | - # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - # linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - # mkdir -p $linux_arm64_dir - # mkdir -p $linux_se_arm64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_arm64_dir)/ - # cp NOTICE.txt $(linux_se_arm64_dir)/ - # displayName: 'Copy required files for packaging' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to signing stage' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - job: Set_3 timeoutInMinutes: 120 @@ -493,195 +493,195 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts condition: succeeded('BuildArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Linux-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # displayName: 'List Artifacts' - - # - script: | - # mkdir mariner && chmod 755 mariner - # cp azCopy-linux-temp/*arm64.rpm mariner - # cp azCopy-linux-temp/*x86_64.rpm mariner - # sudo ls -lRt mariner - # md5sum mariner/* - # displayName: 'Copy artifacts for Mariner' - # workingDirectory: $(Build.ArtifactStagingDirectory) + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + displayName: 'List Artifacts' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: '*.rpm, *.deb, *tar.gz' - # DisplayName: 'ESRP CodeSigning azCopy linux' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, *tar.gz' + DisplayName: 'ESRP CodeSigning azCopy linux' - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/mariner - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - # Pattern: '*.rpm, *.deb' - # DisplayName: 'ESRP CodeSigning azCopy mariner' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - # chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - # rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # md5sum $(Build.ArtifactStagingDirectory)/mariner/* - # displayName: 'List Signed Artifacts' + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - job: Set_2 timeoutInMinutes: 120 @@ -828,186 +828,186 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") - # echo "Folder name: $amd64FolderName" - - # arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") - # echo "Folder name: $arm64FolderName" + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") + echo "Folder name: $amd64FolderName" + + arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") + echo "Folder name: $arm64FolderName" - # # List the contents of the folder - # echo "Contents:" - # ls -l $amd64FolderName - # ls -l $arm64FolderName - - # # Attempt the mv command - # mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # List the contents of the folder + echo "Contents:" + ls -l $amd64FolderName + ls -l $arm64FolderName + + # Attempt the mv command + mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1015,285 +1015,285 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install -y - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install -y + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' - - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install dependencies' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' - - # - job: Set_6 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Find the directory matching the pattern - # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # # Run the azcopy version command - # ./azcopy --version - # ./azcopy --help - # displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_6 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - job: Set_7 timeoutInMinutes: 120 From adbac34839fafc701322ab2467e11c7d9b590ce6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 11:23:49 +0530 Subject: [PATCH 357/513] test linux tar.gz --- release-pipelines.yml | 1026 ++++++++++++++++++++++------------------- 1 file changed, 540 insertions(+), 486 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 58114b23c..dd4e08bb9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -411,184 +411,184 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - displayName: 'Remove arm7 binary from staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + # displayName: 'Remove arm7 binary from staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -683,331 +683,331 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + + # - script: | + # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir -p $windows_amd64 + # mkdir -p $windows_arm64 + # mkdir -p $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") - echo "Folder name: $amd64FolderName" - - arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") - echo "Folder name: $arm64FolderName" + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") + # echo "Folder name: $amd64FolderName" + + # arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") + # echo "Folder name: $arm64FolderName" - # List the contents of the folder - echo "Contents:" - ls -l $amd64FolderName - ls -l $arm64FolderName - - # Attempt the mv command - mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # List the contents of the folder + # echo "Contents:" + # ls -l $amd64FolderName + # ls -l $arm64FolderName + + # # Attempt the mv command + # mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1051,8 +1051,62 @@ stages: azcopy --version azcopy --help displayName: 'Check Version and Help' - + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + + - script: | + # Define the path to the .tar.gz file and the extraction directory + # Define variables + TAR_GZ_FILE=$(ls azcopy*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACTION_DIR" + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 timeoutInMinutes: 120 strategy: matrix: @@ -1091,7 +1145,7 @@ stages: azcopy --help displayName: 'Check Version and Help' - - job: Set_3 + - job: Set_4 timeoutInMinutes: 120 strategy: matrix: @@ -1132,7 +1186,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_4 + - job: Set_5 timeoutInMinutes: 120 strategy: matrix: @@ -1178,7 +1232,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_5 + - job: Set_6 timeoutInMinutes: 120 strategy: matrix: @@ -1224,7 +1278,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_6 + - job: Set_7 timeoutInMinutes: 360 strategy: matrix: @@ -1295,7 +1349,7 @@ stages: ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_7 + - job: Set_8 timeoutInMinutes: 120 strategy: matrix: @@ -1402,7 +1456,7 @@ stages: displayName: 'Extract Files and Run Version and Help Command on Windows' # TestArtifacts ends here - + - stage: ReleaseArtifacts dependsOn: TestArtifacts condition: succeeded('TestArtifacts') From 8ccb1282beb0a0bf40e3740171fb01db1d11ed31 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 12:01:41 +0530 Subject: [PATCH 358/513] test --- release-pipelines.yml | 965 +++++++++++++++++++++--------------------- 1 file changed, 483 insertions(+), 482 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index dd4e08bb9..09c9fdc86 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -411,184 +411,184 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - # displayName: 'Remove arm7 binary from staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -683,331 +683,331 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - - # - script: | - # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir -p $windows_amd64 - # mkdir -p $windows_arm64 - # mkdir -p $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_amd64 -maxdepth 1 -type d -name "azcopy-*") - # echo "Folder name: $amd64FolderName" - - # arm64FolderName=$(find $(Build.ArtifactStagingDirectory)/unzipped_arm64 -maxdepth 1 -type d -name "azcopy-*") - # echo "Folder name: $arm64FolderName" + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy*) + echo "Folder name: $amd64FolderName" + + arm64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy*) + echo "Folder name: $arm64FolderName" - # # List the contents of the folder - # echo "Contents:" - # ls -l $amd64FolderName - # ls -l $arm64FolderName - - # # Attempt the mv command - # mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # List the contents of the folder + echo "Contents:" + ls -l $amd64FolderName + ls -l $arm64FolderName + + # Attempt the mv command + mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -1513,6 +1513,7 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' + workingDirectory: '$(Build.ArtifactStagingDirectory)' #TODO: check once - script: | @@ -1768,7 +1769,7 @@ stages: echo "Package $marinerAmdRpmFile contains 'preview'." while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName done < <(tail -n +5 ../packages.csv) else echo "Package $package_name1 does not contain 'preview'." @@ -1788,7 +1789,7 @@ stages: echo "Skipping for ARM type on $distro" else echo "Publishing for $distro" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName fi done < <(tail -n +5 ../packages.csv) else @@ -1798,7 +1799,7 @@ stages: echo "Skipping for ARM type on $distro" else echo "Publishing for $distro" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName fi done < <(tail -n +3 ../packages.csv | head -n -2) fi From faea26fb61d85c4138b7f9f6e28ef7275a5ac65b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 12:13:22 +0530 Subject: [PATCH 359/513] test update version --- setup/releaseVersionUpdate.py | 67 +++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index c35de9443..739465457 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -13,36 +13,65 @@ print('Incorrect Release Version') sys.exit(1) +# Split the SAS URL to get the container URL and SAS token containerUrl = sasUrl.split('?')[0] sasToken = sasUrl.split('?')[1] -# delete latest version file in the container -# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken -# resp = requests.delete(deleteUrl) -# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') - -# Create a file and write data to it +# Create a file and write the release version to it file_name = 'latest_version.txt' -# Open the file in write mode with open(file_name, 'w') as file: file.write(releaseVersion) print(f'Data written to {file_name}') -# Create the BlobServiceClient using the SAS URL -blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) - -# Get the container name from the SAS URL -container_name = sasUrl.split('?')[0].split('/')[-1] +# Get the full URL to upload the file +putUrl = containerUrl + '/' + file_name + '?' + sasToken -# Get a client to interact with the container -container_client = blob_service_client.get_container_client(container_name) - -# Upload the file to the container +# Upload the file using a PUT request with open(file_name, 'rb') as data: - container_client.upload_blob(name=file_name, data=data, overwrite=True) - -print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + headers = { + 'x-ms-blob-type': 'BlockBlob', + 'Content-Length': str(os.path.getsize(file_name)) + } + resp = requests.put(putUrl, data=data, headers=headers) + +# Check if the request was successful +if resp.status_code < 200 or resp.status_code > 202: + print(f"Failed to upload file: {resp.status_code}, {resp.text}") + sys.exit(1) +else: + print('File successfully uploaded') # Clean up the local file os.remove(file_name) print(f"Local file '{file_name}' deleted.") + +# delete latest version file in the container +# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken +# resp = requests.delete(deleteUrl) +# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') + +# # Create a file and write data to it +# file_name = 'latest_version.txt' +# # Open the file in write mode +# with open(file_name, 'w') as file: +# file.write(releaseVersion) +# print(f'Data written to {file_name}') + +# # Create the BlobServiceClient using the SAS URL +# blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) + +# # Get the container name from the SAS URL +# container_name = sasUrl.split('?')[0].split('/')[-1] + +# # Get a client to interact with the container +# container_client = blob_service_client.get_container_client(container_name) + +# # Upload the file to the container +# with open(file_name, 'rb') as data: +# container_client.upload_blob(name=file_name, data=data, overwrite=True) + +# print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + +# # Clean up the local file +# os.remove(file_name) +# print(f"Local file '{file_name}' deleted.") From bd408b71ce766ab51434787600ce1de3c6c3b01a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 12:15:13 +0530 Subject: [PATCH 360/513] test update version --- release-pipelines.yml | 1826 ++++++++++++++++++++--------------------- 1 file changed, 913 insertions(+), 913 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 09c9fdc86..4056534b1 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -411,184 +411,184 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - displayName: 'Remove arm7 binary from staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + # displayName: 'Remove arm7 binary from staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -683,779 +683,779 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + + # - script: | + # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir -p $windows_amd64 + # mkdir -p $windows_arm64 + # mkdir -p $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy*) - echo "Folder name: $amd64FolderName" - - arm64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy*) - echo "Folder name: $arm64FolderName" + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy*) + # echo "Folder name: $amd64FolderName" + + # arm64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy*) + # echo "Folder name: $arm64FolderName" - # List the contents of the folder - echo "Contents:" - ls -l $amd64FolderName - ls -l $arm64FolderName - - # Attempt the mv command - mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # List the contents of the folder + # echo "Contents:" + # ls -l $amd64FolderName + # ls -l $arm64FolderName + + # # Attempt the mv command + # mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_linux.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + # Pattern: 'azcopy_linux*' + # DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - stage: TestArtifacts + # dependsOn: SignArtifacts + # condition: succeeded('SignArtifacts') + # jobs: + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_tar_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - - - script: | - # Define the path to the .tar.gz file and the extraction directory - # Define variables - TAR_GZ_FILE=$(ls azcopy*.tar.gz) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the .tar.gz file - tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACTION_DIR" - - # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - echo "Found matching directory: ${matching_dirs[0]}" - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - ./azcopy --version - ./azcopy --help - displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install -y - ldd --version - displayName: "GLIBC Version" - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_tar_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + + # - script: | + # # Define the path to the .tar.gz file and the extraction directory + # # Define variables + # TAR_GZ_FILE=$(ls azcopy*.tar.gz) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # # Create extraction directory + # mkdir -p $EXTRACT_DIR + + # # Extract the .tar.gz file + # tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACTION_DIR" + + # # Find the directory matching the pattern + # matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # echo "Found matching directory: ${matching_dirs[0]}" + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi + + # ./azcopy --version + # ./azcopy --help + # displayName: 'Check Version and Help' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install -y + # ldd --version + # displayName: "GLIBC Version" + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' + + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' + + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_6 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install dependencies' + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' + + # - job: Set_6 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" + + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # steps: + # - checkout: none + + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install dependencies' - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_7 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi - - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run the azcopy version command - ./azcopy --version - ./azcopy --help - displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # sudo rpm -qip azcopy*x86_64.rpm + # sudo rpm -i azcopy*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' + + # - job: Set_7 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi + + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # # Create extraction directory + # mkdir -p $EXTRACT_DIR + + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # # Find the directory matching the pattern + # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi + + # # Run the azcopy version command + # ./azcopy --version + # ./azcopy --help + # displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_8 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_8 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Find directories matching the pattern - $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # Check if there is exactly one matching directory - if ($matchingDirs.Count -eq 0) { - Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } elseif ($matchingDirs.Count -gt 1) { - Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } - - # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" - if (-Not (Test-Path -Path $azcopyPath)) { - Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - exit 1 - } else { - Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # Ensure the azcopy executable has the correct permissions - icacls $azcopyPath - - # Run azcopy --version and capture output - try { - $versionOutput = & $azcopyPath --version 2>&1 - Write-Output "azcopy version output: $versionOutput" - } catch { - Write-Error "Failed to run azcopy --version: $_" - } - - # Run azcopy --help and capture output - try { - $helpOutput = & $azcopyPath --help 2>&1 - Write-Output "azcopy help output: $helpOutput" - } catch { - Write-Error "Failed to run azcopy --help: $_" - } - } - displayName: 'Extract Files and Run Version and Help Command on Windows' + # # Find the zip file matching the pattern + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } + + # $zipFilePath = $zipFile.FullName + + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } + + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir + + # # Find directories matching the pattern + # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # # Check if there is exactly one matching directory + # if ($matchingDirs.Count -eq 0) { + # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } elseif ($matchingDirs.Count -gt 1) { + # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } + + # # Check if the azcopy executable exists in the matching directory + # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + # if (-Not (Test-Path -Path $azcopyPath)) { + # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + # exit 1 + # } else { + # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # # Ensure the azcopy executable has the correct permissions + # icacls $azcopyPath + + # # Run azcopy --version and capture output + # try { + # $versionOutput = & $azcopyPath --version 2>&1 + # Write-Output "azcopy version output: $versionOutput" + # } catch { + # Write-Error "Failed to run azcopy --version: $_" + # } + + # # Run azcopy --help and capture output + # try { + # $helpOutput = & $azcopyPath --help 2>&1 + # Write-Output "azcopy help output: $helpOutput" + # } catch { + # Write-Error "Failed to run azcopy --help: $_" + # } + # } + # displayName: 'Extract Files and Run Version and Help Command on Windows' - # TestArtifacts ends here + # # TestArtifacts ends here - stage: ReleaseArtifacts dependsOn: TestArtifacts From 90d5164c7fb363e0476f2d62c9438129fcb28f92 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 12:16:50 +0530 Subject: [PATCH 361/513] test update version --- release-pipelines.yml | 634 +++++++++++++++++++++--------------------- 1 file changed, 317 insertions(+), 317 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4056534b1..d44a94ed2 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1457,355 +1457,355 @@ stages: # # TestArtifacts ends here - - stage: ReleaseArtifacts - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseToContainer - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) + # - stage: ReleaseArtifacts + # dependsOn: TestArtifacts + # condition: succeeded('TestArtifacts') + # jobs: + # - job: ReleaseToContainer + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) - variables: - - name: Test_Release_Account_Name - value: 'azcopyprivatedrops' - - name: Release_Account_Name - value: 'azcopyvnextrelease' - - name: Test_Mode - value: ${{ parameters.test_mode }} + # variables: + # - name: Test_Release_Account_Name + # value: 'azcopyprivatedrops' + # - name: Release_Account_Name + # value: 'azcopyvnextrelease' + # - name: Test_Mode + # value: ${{ parameters.test_mode }} - steps: + # steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Linux Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Windows Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Mac Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Binaries Build Artifacts' - inputs: - artifactName: 'azCopy-binaries-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Binaries Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - cp NOTICE.txt azCopy-binaries-signed/ - sudo apt-get install -y zip - cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - #TODO: check once - - script: | - cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed - echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" - ls -l + # - script: | + # cp NOTICE.txt azCopy-binaries-signed/ + # sudo apt-get install -y zip + # cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Create drop.zip file and install azcopy deb package' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # #TODO: check once + # - script: | + # cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + # echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" + # ls -l - # Create the zip archive with all files inside azCopy-binaries-signed - zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + # # Create the zip archive with all files inside azCopy-binaries-signed + # zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - # List the contents of the resulting zip file for verification - unzip -l $(Build.ArtifactStagingDirectory)/drop.zip - displayName: 'Archive all binaries to drop.zip' + # # List the contents of the resulting zip file for verification + # unzip -l $(Build.ArtifactStagingDirectory)/drop.zip + # displayName: 'Archive all binaries to drop.zip' - - task: Bash@3 - displayName: 'Install Azure CLI' - inputs: - targetType: 'inline' - script: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # - task: Bash@3 + # displayName: 'Install Azure CLI' + # inputs: + # targetType: 'inline' + # script: | + # curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - task: AzureCLI@2 - displayName: 'Azure CLI' - inputs: - azureSubscription: 'ESRP KeyVault identity' - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - # Find the current version - raw_version_output=$(azcopy --version) - azcopy_version=$(echo $raw_version_output | awk '{print $3}') - - today=$(date +"%Y%m%d") + # - task: AzureCLI@2 + # displayName: 'Azure CLI' + # inputs: + # azureSubscription: 'ESRP KeyVault identity' + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # # Find the current version + # raw_version_output=$(azcopy --version) + # azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + # today=$(date +"%Y%m%d") - if [ $(Test_Mode) = "True" ]; then - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - else - container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - fi - echo "Artifacts will be uploaded to: $container_url" + # if [ $(Test_Mode) = "True" ]; then + # container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + # else + # container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + # fi + # echo "Artifacts will be uploaded to: $container_url" - executable_name=azcopy + # executable_name=azcopy - # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + # # Upload the archived builds + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - - job: ReleaseAzcopy - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: ReleaseAzcopy + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu-22: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Linux Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Windows Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Mac Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt update + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-3.1 -y + # displayName: "Update dependencies" - - ${{ if eq(parameters.post_release, true) }}: + # - ${{ if eq(parameters.post_release, true) }}: - # add release tags & push to github - #TODO: update github connection and repo name while merging to main - - task: GithubRelease@1 - inputs: - githubConnection: 'dphulkar-msft-github-rel' - repositoryName: 'dphulkar-msft/azure-storage-azcopy' - action: 'edit' + # # add release tags & push to github + # #TODO: update github connection and repo name while merging to main + # - task: GithubRelease@1 + # inputs: + # githubConnection: 'dphulkar-msft-github-rel' + # repositoryName: 'dphulkar-msft/azure-storage-azcopy' + # action: 'edit' - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' + # target: '$(Build.SourceVersion)' + # tagSource: 'userSpecifiedTag' - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} + # title: ${{ parameters.tag }} + # tag: ${{ parameters.tag }} - assets: | - $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace - - - ${{ if eq(parameters.publish_artifacts, true) }}: - - stage: PublishArtifacts - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: PublishArtifacts - timeoutInMinutes: 120 - pool: - vmImage: 'ubuntu-22.04' - variables: - - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # assets: | + # $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + # $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + # $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + # changeLogCompareToRelease: 'lastFullRelease' + # changeLogType: 'commitBased' + # isDraft: ${{ parameters.draft }} + # isPreRelease: ${{ parameters.prerelease }} + # assetUploadMode: replace + + # - ${{ if eq(parameters.publish_artifacts, true) }}: + # - stage: PublishArtifacts + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') + # jobs: + # - job: PublishArtifacts + # timeoutInMinutes: 120 + # pool: + # vmImage: 'ubuntu-22.04' + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - checkout: none - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'DevExGlobalFeed' - displayName: 'Connect to PMC artifact' - - - script: | - pip install pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Signed Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + # steps: + # - checkout: none + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'DevExGlobalFeed' + # displayName: 'Connect to PMC artifact' + + # - script: | + # pip install pmc-cli + # displayName: 'Install pmc-cli' + + # - task: DownloadSecureFile@1 + # name: pmcCertificate + # displayName: 'Download pmc pem file' + # inputs: + # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + # - task: DownloadSecureFile@1 + # name: settings + # displayName: 'Download settings.toml file' + # inputs: + # secureFile: 'settings.toml' + + # - script: | + # pmc --version + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + # if [ $? -ne 0 ]; then + # exit 1 + # fi + # displayName: 'Test PMC installation' + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Signed Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - cd mariner - arm64file=$(ls azcopy*.arm64.rpm) - amd64file=$(ls azcopy*.x86_64.rpm) - marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" - marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" - mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - sudo ls -lRt - mv azcopy*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - displayName: 'List Artifacts' - - - script: | - mv azCopy-linux-signed/*tar.gz . - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed - displayName: 'Upload files' - workingDirectory: $(Build.ArtifactStagingDirectory) - - # TODO: replace org name while merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv - cat ./packages.csv - displayName: 'Fetch packages.csv' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - echo "Azcopy AMD DEB $azcopyAmdDeb" - echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - - azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy AMD RPM $azcopyAmdRpmFile" - echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + # - script: | + # cd mariner + # arm64file=$(ls azcopy*.arm64.rpm) + # amd64file=$(ls azcopy*.x86_64.rpm) + # marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + # marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + # mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + # mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + # sudo ls -lRt + # mv azcopy*.rpm .. + # cd .. + # rm -r mariner/ + # displayName: 'Rename Mariner binaries' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # displayName: 'List Artifacts' + + # - script: | + # mv azCopy-linux-signed/*tar.gz . + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + # displayName: 'Upload files' + # workingDirectory: $(Build.ArtifactStagingDirectory) + + # # TODO: replace org name while merging to main + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + # cat ./packages.csv + # displayName: 'Fetch packages.csv' + # workingDirectory: $(Build.ArtifactStagingDirectory)/ + + # - script: | + # azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + # echo "Azcopy AMD DEB $azcopyAmdDeb" + # echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + # azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + # echo "Azcopy AMD RPM $azcopyAmdRpmFile" + # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - echo "Azcopy ARM DEB $azcopyArmDeb" - echo "Azcopy ARM DEB ID: $azcopyArmDeb" - - azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - echo "Azcopy ARM RPM $azcopyArmRpmFile" - echo "Azcopy ARM RPM ID: $azcopyArmRpm" - - # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + # echo "Azcopy ARM DEB $azcopyArmDeb" + # echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + # azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') + # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + # echo "Azcopy ARM RPM $azcopyArmRpmFile" + # echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # # Find the Mariner RPM file and upload + # marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + # marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" - is_preview="false" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - - if [[ $marinerAmdRpmFile == *"preview"* ]]; then - is_preview="true" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - - echo "Package $marinerAmdRpmFile contains 'preview'." - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +5 ../packages.csv) - else - echo "Package $package_name1 does not contain 'preview'." - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv | head -n -2) - fi - displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - - script: | - if [ "$(is_preview)" = "true" ]; then - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +5 ../packages.csv) - else - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv | head -n -2) - fi + # is_preview="false" + # echo "##vso[task.setvariable variable=is_preview]$is_preview" + + # if [[ $marinerAmdRpmFile == *"preview"* ]]; then + # is_preview="true" + # echo "##vso[task.setvariable variable=is_preview]$is_preview" + + # echo "Package $marinerAmdRpmFile contains 'preview'." + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +5 ../packages.csv) + # else + # echo "Package $package_name1 does not contain 'preview'." + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # echo "Uploading packages for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + # done < <(tail -n +3 ../packages.csv | head -n -2) + # fi + # displayName: 'Add uploaded packages to repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + # - script: | + # if [ "$(is_preview)" = "true" ]; then + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +5 ../packages.csv) + # else + # while IFS=, read -r distro fuseArchType repoName releaseName; do + # if [[ $fuseArchType == *"Arm"* ]] + # then + # echo "Skipping for ARM type on $distro" + # else + # echo "Publishing for $distro" + # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + # fi + # done < <(tail -n +3 ../packages.csv | head -n -2) + # fi - displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + # displayName: 'Publish the repository' + # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion dependsOn: ReleaseArtifacts From a8590932459a3e144f79973d6673e3c24dbf6b1f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 12:17:43 +0530 Subject: [PATCH 362/513] test update version --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d44a94ed2..d465a7698 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1808,8 +1808,8 @@ stages: # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: From bd7f29c94ee2ddf766dc97760b22e1e29d4561fe Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 13:03:39 +0530 Subject: [PATCH 363/513] test tar.gz and mac unzip --- release-pipelines.yml | 1698 ++++++++++++++++----------------- setup/releaseVersionUpdate.py | 33 +- 2 files changed, 850 insertions(+), 881 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d465a7698..131a52f68 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -828,109 +828,109 @@ stages: # artifactName: 'azCopy-windows-signed' # displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET + variables: + - group: AZCOPY_ESRP_SECRET - # steps: - # - checkout: none + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' - # - script: | - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - # unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 + mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 + unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - # # Use find to retrieve the folder name - # amd64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy*) - # echo "Folder name: $amd64FolderName" + # Use find to retrieve the folder name + amd64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy_darwin_amd64*) + echo "Folder name: $amd64FolderName" - # arm64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy*) - # echo "Folder name: $arm64FolderName" + arm64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy_darwin_arm64*) + echo "Folder name: $arm64FolderName" - # # List the contents of the folder - # echo "Contents:" - # ls -l $amd64FolderName - # ls -l $arm64FolderName + # List the contents of the folder + echo "Contents:" + ls -l $amd64FolderName + ls -l $arm64FolderName - # # Attempt the mv command - # mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # Attempt the mv command + mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' # - job: Set_4 # timeoutInMinutes: 120 @@ -1011,801 +1011,801 @@ stages: # SignArtifacts end here - # - stage: TestArtifacts - # dependsOn: SignArtifacts - # condition: succeeded('SignArtifacts') - # jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_tar_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - - # - script: | - # # Define the path to the .tar.gz file and the extraction directory - # # Define variables - # TAR_GZ_FILE=$(ls azcopy*.tar.gz) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the .tar.gz file - # tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACTION_DIR" - - # # Find the directory matching the pattern - # matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # echo "Found matching directory: ${matching_dirs[0]}" - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # ./azcopy --version - # ./azcopy --help - # displayName: 'Check Version and Help' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install -y - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + + - script: | + # Define the path to the .tar.gz file and the extraction directory + # Define variables + TAR_GZ_FILE=$(ls azcopy*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install -y + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' - - # - job: Set_6 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install dependencies' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_6 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' - - # - job: Set_7 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Find the directory matching the pattern - # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # # Run the azcopy version command - # ./azcopy --version - # ./azcopy --help - # displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_7 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - # - job: Set_8 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_8 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # # Find the zip file matching the pattern - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } - - # $zipFilePath = $zipFile.FullName - - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } - - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir - - # # Find directories matching the pattern - # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # # Check if there is exactly one matching directory - # if ($matchingDirs.Count -eq 0) { - # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } elseif ($matchingDirs.Count -gt 1) { - # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } - - # # Check if the azcopy executable exists in the matching directory - # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" - # if (-Not (Test-Path -Path $azcopyPath)) { - # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - # exit 1 - # } else { - # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # # Ensure the azcopy executable has the correct permissions - # icacls $azcopyPath - - # # Run azcopy --version and capture output - # try { - # $versionOutput = & $azcopyPath --version 2>&1 - # Write-Output "azcopy version output: $versionOutput" - # } catch { - # Write-Error "Failed to run azcopy --version: $_" - # } - - # # Run azcopy --help and capture output - # try { - # $helpOutput = & $azcopyPath --help 2>&1 - # Write-Output "azcopy help output: $helpOutput" - # } catch { - # Write-Error "Failed to run azcopy --help: $_" - # } - # } - # displayName: 'Extract Files and Run Version and Help Command on Windows' + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } + } + displayName: 'Extract Files and Run Version and Help Command on Windows' - # # TestArtifacts ends here - - # - stage: ReleaseArtifacts - # dependsOn: TestArtifacts - # condition: succeeded('TestArtifacts') - # jobs: - # - job: ReleaseToContainer - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) + # TestArtifacts ends here + + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToContainer + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) - # variables: - # - name: Test_Release_Account_Name - # value: 'azcopyprivatedrops' - # - name: Release_Account_Name - # value: 'azcopyvnextrelease' - # - name: Test_Mode - # value: ${{ parameters.test_mode }} - - # steps: - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Linux Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Windows Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Mac Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + - name: Test_Mode + value: ${{ parameters.test_mode }} + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Binaries Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # cp NOTICE.txt azCopy-binaries-signed/ - # sudo apt-get install -y zip - # cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Create drop.zip file and install azcopy deb package' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # #TODO: check once - # - script: | - # cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed - # echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" - # ls -l + - task: DownloadBuildArtifacts@0 + displayName: 'Download Binaries Build Artifacts' + inputs: + artifactName: 'azCopy-binaries-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cp NOTICE.txt azCopy-binaries-signed/ + sudo apt-get install -y zip + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Create drop.zip file and install azcopy deb package' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + #TODO: check once + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" + ls -l - # # Create the zip archive with all files inside azCopy-binaries-signed - # zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + # Create the zip archive with all files inside azCopy-binaries-signed + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - # # List the contents of the resulting zip file for verification - # unzip -l $(Build.ArtifactStagingDirectory)/drop.zip - # displayName: 'Archive all binaries to drop.zip' - - # - task: Bash@3 - # displayName: 'Install Azure CLI' - # inputs: - # targetType: 'inline' - # script: | - # curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - # - task: AzureCLI@2 - # displayName: 'Azure CLI' - # inputs: - # azureSubscription: 'ESRP KeyVault identity' - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # # Find the current version - # raw_version_output=$(azcopy --version) - # azcopy_version=$(echo $raw_version_output | awk '{print $3}') - - # today=$(date +"%Y%m%d") + # List the contents of the resulting zip file for verification + unzip -l $(Build.ArtifactStagingDirectory)/drop.zip + displayName: 'Archive all binaries to drop.zip' + + - task: Bash@3 + displayName: 'Install Azure CLI' + inputs: + targetType: 'inline' + script: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=$(azcopy --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + today=$(date +"%Y%m%d") - # if [ $(Test_Mode) = "True" ]; then - # container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - # else - # container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - # fi - # echo "Artifacts will be uploaded to: $container_url" + if [ $(Test_Mode) = "True" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" - # executable_name=azcopy + executable_name=azcopy - # # Upload the archived builds - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - # AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - - # - job: ReleaseAzcopy - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu-22: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Linux Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Windows Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Mac Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip - # displayName: 'List Artifacts' - - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt update - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-3.1 -y - # displayName: "Update dependencies" + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" - # - ${{ if eq(parameters.post_release, true) }}: + - ${{ if eq(parameters.post_release, true) }}: - # # add release tags & push to github - # #TODO: update github connection and repo name while merging to main - # - task: GithubRelease@1 - # inputs: - # githubConnection: 'dphulkar-msft-github-rel' - # repositoryName: 'dphulkar-msft/azure-storage-azcopy' - # action: 'edit' + # add release tags & push to github + #TODO: update github connection and repo name while merging to main + - task: GithubRelease@1 + inputs: + githubConnection: 'dphulkar-msft-github-rel' + repositoryName: 'dphulkar-msft/azure-storage-azcopy' + action: 'edit' - # target: '$(Build.SourceVersion)' - # tagSource: 'userSpecifiedTag' + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' - # title: ${{ parameters.tag }} - # tag: ${{ parameters.tag }} + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} - # assets: | - # $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* - # $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* - # $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* - # changeLogCompareToRelease: 'lastFullRelease' - # changeLogType: 'commitBased' - # isDraft: ${{ parameters.draft }} - # isPreRelease: ${{ parameters.prerelease }} - # assetUploadMode: replace - - # - ${{ if eq(parameters.publish_artifacts, true) }}: - # - stage: PublishArtifacts - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') - # jobs: - # - job: PublishArtifacts - # timeoutInMinutes: 120 - # pool: - # vmImage: 'ubuntu-22.04' - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + assets: | + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - checkout: none - - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'DevExGlobalFeed' - # displayName: 'Connect to PMC artifact' - - # - script: | - # pip install pmc-cli - # displayName: 'Install pmc-cli' - - # - task: DownloadSecureFile@1 - # name: pmcCertificate - # displayName: 'Download pmc pem file' - # inputs: - # secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - # - task: DownloadSecureFile@1 - # name: settings - # displayName: 'Download settings.toml file' - # inputs: - # secureFile: 'settings.toml' - - # - script: | - # pmc --version - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - # if [ $? -ne 0 ]; then - # exit 1 - # fi - # displayName: 'Test PMC installation' - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Signed Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # cd mariner - # arm64file=$(ls azcopy*.arm64.rpm) - # amd64file=$(ls azcopy*.x86_64.rpm) - # marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" - # marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" - # mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - # mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - # sudo ls -lRt - # mv azcopy*.rpm .. - # cd .. - # rm -r mariner/ - # displayName: 'Rename Mariner binaries' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # displayName: 'List Artifacts' - - # - script: | - # mv azCopy-linux-signed/*tar.gz . - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed - # displayName: 'Upload files' - # workingDirectory: $(Build.ArtifactStagingDirectory) - - # # TODO: replace org name while merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv - # cat ./packages.csv - # displayName: 'Fetch packages.csv' - # workingDirectory: $(Build.ArtifactStagingDirectory)/ - - # - script: | - # azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - # echo "Azcopy AMD DEB $azcopyAmdDeb" - # echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - - # azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - # echo "Azcopy AMD RPM $azcopyAmdRpmFile" - # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + - script: | + cd mariner + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + sudo ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + mv azCopy-linux-signed/*tar.gz . + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + displayName: 'Upload files' + workingDirectory: $(Build.ArtifactStagingDirectory) + + # TODO: replace org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB $azcopyAmdDeb" + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy AMD RPM $azcopyAmdRpmFile" + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - # echo "Azcopy ARM DEB $azcopyArmDeb" - # echo "Azcopy ARM DEB ID: $azcopyArmDeb" - - # azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - # echo "Azcopy ARM RPM $azcopyArmRpmFile" - # echo "Azcopy ARM RPM ID: $azcopyArmRpm" - - # # Find the Mariner RPM file and upload - # marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - # marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB $azcopyArmDeb" + echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" + echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" - # is_preview="false" - # echo "##vso[task.setvariable variable=is_preview]$is_preview" - - # if [[ $marinerAmdRpmFile == *"preview"* ]]; then - # is_preview="true" - # echo "##vso[task.setvariable variable=is_preview]$is_preview" - - # echo "Package $marinerAmdRpmFile contains 'preview'." - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +5 ../packages.csv) - # else - # echo "Package $package_name1 does not contain 'preview'." - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # echo "Uploading packages for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - # done < <(tail -n +3 ../packages.csv | head -n -2) - # fi - # displayName: 'Add uploaded packages to repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - # - script: | - # if [ "$(is_preview)" = "true" ]; then - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +5 ../packages.csv) - # else - # while IFS=, read -r distro fuseArchType repoName releaseName; do - # if [[ $fuseArchType == *"Arm"* ]] - # then - # echo "Skipping for ARM type on $distro" - # else - # echo "Publishing for $distro" - # pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - # fi - # done < <(tail -n +3 ../packages.csv | head -n -2) - # fi + is_preview="false" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + is_preview="true" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + + echo "Package $marinerAmdRpmFile contains 'preview'." + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +5 ../packages.csv) + else + echo "Package $package_name1 does not contain 'preview'." + while IFS=, read -r distro fuseArchType repoName releaseName; do + echo "Uploading packages for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv | head -n -2) + fi + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + if [ "$(is_preview)" = "true" ]; then + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +5 ../packages.csv) + else + while IFS=, read -r distro fuseArchType repoName releaseName; do + if [[ $fuseArchType == *"Arm"* ]] + then + echo "Skipping for ARM type on $distro" + else + echo "Publishing for $distro" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv | head -n -2) + fi - # displayName: 'Publish the repository' - # workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion # dependsOn: ReleaseArtifacts diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index 739465457..f994cdfce 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -43,35 +43,4 @@ # Clean up the local file os.remove(file_name) -print(f"Local file '{file_name}' deleted.") - -# delete latest version file in the container -# deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken -# resp = requests.delete(deleteUrl) -# sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') - -# # Create a file and write data to it -# file_name = 'latest_version.txt' -# # Open the file in write mode -# with open(file_name, 'w') as file: -# file.write(releaseVersion) -# print(f'Data written to {file_name}') - -# # Create the BlobServiceClient using the SAS URL -# blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) - -# # Get the container name from the SAS URL -# container_name = sasUrl.split('?')[0].split('/')[-1] - -# # Get a client to interact with the container -# container_client = blob_service_client.get_container_client(container_name) - -# # Upload the file to the container -# with open(file_name, 'rb') as data: -# container_client.upload_blob(name=file_name, data=data, overwrite=True) - -# print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") - -# # Clean up the local file -# os.remove(file_name) -# print(f"Local file '{file_name}' deleted.") +print(f"Local file '{file_name}' deleted.") \ No newline at end of file From 0b49bf6320c377985611bcb835d9a96f2c17f1ec Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 13:06:03 +0530 Subject: [PATCH 364/513] test tar.gz and mac unzip --- release-pipelines.yml | 784 +++++++++++++++++++++--------------------- 1 file changed, 392 insertions(+), 392 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 131a52f68..0376899a0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -493,102 +493,102 @@ stages: # verbosity: 'Verbose' # alertWarningLevel: 'High' - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -1106,356 +1106,356 @@ stages: displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22-arm64' + # AgentName: "blobfuse-ubn22-arm64" + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(AgentName) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install -y - ldd --version - displayName: "GLIBC Version" + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install -y + # ldd --version + # displayName: "GLIBC Version" - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - script: | - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - script: | + # sudo dpkg --info azcopy-*arm64.deb + # sudo dpkg -i azcopy-*arm64.deb + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Arm64: + # agentName: "blobfuse-rhel9-arm64" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9-arm64' - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*.arm64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*.arm64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' + # - job: Set_5 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Rhel_Amd64: + # agentName: "blobfuse-rhel9" + # vmImage: 'RHEL-9.0' + # container: 'test-cnt-rhel-9' - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) + # pool: + # name: "blobfuse-rhel-pool" + # demands: + # - ImageOverride -equals $(agentName) - steps: - - checkout: none - - script: | - sudo yum update -y - sudo yum install git -y - displayName: 'Install Git' + # steps: + # - checkout: none + # - script: | + # sudo yum update -y + # sudo yum install git -y + # displayName: 'Install Git' - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - job: Set_6 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" + # - script: | + # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + # sudo rpm -qip azcopy-*x86_64.rpm + # sudo yum groupinstall "Development Tools" -y + # sudo rpm -i azcopy-*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' + + # - job: Set_6 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mariner2: + # agentName: "blobfuse-mariner2" + # DistroVer: "Mariner2" + # Description: "CBL-Mariner2 Linux" + # container: "test-cnt-mari-2" - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) + # pool: + # name: "blobfuse-mariner-pool" + # demands: + # - ImageOverride -equals $(AgentName) - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - sudo tdnf update -y - sudo tdnf install git -y - displayName: 'Install dependencies' + # - script: | + # sudo tdnf update -y + # sudo tdnf install git -y + # displayName: 'Install dependencies' - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" - - job: Set_7 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - script: | + # sudo rpm -qip azcopy*x86_64.rpm + # sudo rpm -i azcopy*x86_64.rpm + # displayName: 'Install Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help command' + + # - job: Set_7 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi + # - script: | + # # Install Homebrew if not already installed + # if ! command -v brew &> /dev/null + # then + # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + # eval "$(/opt/homebrew/bin/brew shellenv)" + # fi - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # # Define variables + # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Create extraction directory - mkdir -p $EXTRACT_DIR + # # Create extraction directory + # mkdir -p $EXTRACT_DIR - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + # # Extract the zip file + # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + # # Find the directory matching the pattern + # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi - # Run the azcopy version command - ./azcopy --version - ./azcopy --help - displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + # # Run the azcopy version command + # ./azcopy --version + # ./azcopy --help + # displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_8 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) + # - job: Set_8 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-signed' + # downloadPath: $(Build.ArtifactStagingDirectory) + # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + # - task: PowerShell@2 + # inputs: + # targetType: 'inline' + # script: | + # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Find directories matching the pattern - $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # Check if there is exactly one matching directory - if ($matchingDirs.Count -eq 0) { - Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } elseif ($matchingDirs.Count -gt 1) { - Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } - - # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" - if (-Not (Test-Path -Path $azcopyPath)) { - Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - exit 1 - } else { - Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # Ensure the azcopy executable has the correct permissions - icacls $azcopyPath - - # Run azcopy --version and capture output - try { - $versionOutput = & $azcopyPath --version 2>&1 - Write-Output "azcopy version output: $versionOutput" - } catch { - Write-Error "Failed to run azcopy --version: $_" - } - - # Run azcopy --help and capture output - try { - $helpOutput = & $azcopyPath --help 2>&1 - Write-Output "azcopy help output: $helpOutput" - } catch { - Write-Error "Failed to run azcopy --help: $_" - } - } - displayName: 'Extract Files and Run Version and Help Command on Windows' + # # Find the zip file matching the pattern + # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + # if ($null -eq $zipFile) { + # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + # exit 1 + # } + + # $zipFilePath = $zipFile.FullName + + # # Create extraction directory + # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # # Extract the zip file + # try { + # Add-Type -AssemblyName System.IO.Compression.FileSystem + # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + # } catch { + # Write-Error "Failed to extract zip file: $_" + # exit 1 + # } + + # # Change to the directory containing azcopy + # Set-Location -Path $extractDir + + # # Find directories matching the pattern + # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # # Check if there is exactly one matching directory + # if ($matchingDirs.Count -eq 0) { + # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } elseif ($matchingDirs.Count -gt 1) { + # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + # exit 1 + # } + + # # Check if the azcopy executable exists in the matching directory + # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + # if (-Not (Test-Path -Path $azcopyPath)) { + # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + # exit 1 + # } else { + # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # # Ensure the azcopy executable has the correct permissions + # icacls $azcopyPath + + # # Run azcopy --version and capture output + # try { + # $versionOutput = & $azcopyPath --version 2>&1 + # Write-Output "azcopy version output: $versionOutput" + # } catch { + # Write-Error "Failed to run azcopy --version: $_" + # } + + # # Run azcopy --help and capture output + # try { + # $helpOutput = & $azcopyPath --help 2>&1 + # Write-Output "azcopy help output: $helpOutput" + # } catch { + # Write-Error "Failed to run azcopy --help: $_" + # } + # } + # displayName: 'Extract Files and Run Version and Help Command on Windows' - # TestArtifacts ends here + # # TestArtifacts ends here - stage: ReleaseArtifacts dependsOn: TestArtifacts From c7fdd6306e99b2bdeba7b961fc574906164a4284 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 13:37:33 +0530 Subject: [PATCH 365/513] test tar.gz and mac unzip --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0376899a0..c63298c22 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1080,7 +1080,7 @@ stages: - script: | # Define the path to the .tar.gz file and the extraction directory # Define variables - TAR_GZ_FILE=$(ls azcopy*.tar.gz) + TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" # Create extraction directory From 85a29debe0d588f404856098fcc7239211fdb858 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 13:46:15 +0530 Subject: [PATCH 366/513] test tar.gz and mac unzip --- release-pipelines.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c63298c22..5c0294f81 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -899,17 +899,17 @@ stages: displayName: 'Publish Signed Artifacts' - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 # Use find to retrieve the folder name - amd64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy_darwin_amd64*) - echo "Folder name: $amd64FolderName" + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" - arm64FolderName=$(ls $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy_darwin_arm64*) - echo "Folder name: $arm64FolderName" + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" # List the contents of the folder echo "Contents:" @@ -917,14 +917,15 @@ stages: ls -l $arm64FolderName # Attempt the mv command - mv $amd64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $arm64FolderName/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + mv $amd64FolderName/azcopy azcopy_darwin_amd64 + mv $arm64FolderName/azcopy azcopy_darwin_arm64 rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 From fe8441f431c753ba6a2bc00015adc34a9a3a9c6e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 15:00:32 +0530 Subject: [PATCH 367/513] test tar.gz and mac unzip --- release-pipelines.yml | 199 +++++++++++++++++++++++------------------- 1 file changed, 109 insertions(+), 90 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5c0294f81..1df440327 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -828,110 +828,100 @@ stages: # artifactName: 'azCopy-windows-signed' # displayName: 'Publish Signed Artifacts' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) - variables: - - group: AZCOPY_ESRP_SECRET + # variables: + # - group: AZCOPY_ESRP_SECRET - steps: - - checkout: none + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p unzipped_amd64 - mkdir -p unzipped_arm64 - unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - echo "AMD64 Folder name: $amd64FolderName" + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' - arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - # List the contents of the folder - echo "Contents:" - ls -l $amd64FolderName - ls -l $arm64FolderName + # - script: | + # mkdir -p unzipped_amd64 + # mkdir -p unzipped_arm64 + # unzip -o azcopy*amd64*.zip -d unzipped_amd64 + # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - # Attempt the mv command - mv $amd64FolderName/azcopy azcopy_darwin_amd64 - mv $arm64FolderName/azcopy azcopy_darwin_arm64 + # # Use find to retrieve the folder name + # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + # mv $amd64FolderName/azcopy azcopy_darwin_amd64 + # mv $arm64FolderName/azcopy azcopy_darwin_arm64 - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' # - job: Set_4 # timeoutInMinutes: 120 @@ -1079,16 +1069,37 @@ stages: item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - script: | + #!/bin/bash + # Define the path to the .tar.gz file and the extraction directory # Define variables TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # Check if the file exists and is a regular file + if [ ! -f "$TAR_GZ_FILE" ]; then + echo "Error: File $TAR_GZ_FILE does not exist." + exit 1 + fi + + # Verify the file type + file_type=$(file -b "$TAR_GZ_FILE") + echo "File type: $file_type" + + if [[ "$file_type" != *"gzip compressed data"* ]]; then + echo "Error: The file is not a valid .tar.gz file" + exit 1 + fi + # Create extraction directory - mkdir -p $EXTRACT_DIR + mkdir -p "$EXTRACT_DIR" # Extract the .tar.gz file tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi # Find the directory matching the pattern matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) @@ -1102,8 +1113,16 @@ stages: exit 1 fi + # Check if azcopy exists and is executable + if [ ! -x "./azcopy" ]; then + echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" + exit 1 + fi + + # Run azcopy commands ./azcopy --version ./azcopy --help + displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed From 362eac60a81ad507a12ce9cf70eac7f50bc7d7ec Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 17:20:03 +0530 Subject: [PATCH 368/513] test unsigned tar.gz --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1df440327..77bf6089c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1064,7 +1064,7 @@ stages: - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' + artifact_name: 'azCopy-linux-temp' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' @@ -1124,7 +1124,7 @@ stages: ./azcopy --help displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-temp # - job: Set_3 # timeoutInMinutes: 120 From c195704277f39c81952ff695ae1dc2d7b8298cd0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 17:36:53 +0530 Subject: [PATCH 369/513] test unsigned tar.gz --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 77bf6089c..9cc145eea 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1066,7 +1066,7 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-temp' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + item_pattern: 'azCopy-linux-temp/azcopy_linux_amd64*.tar.gz' - script: | #!/bin/bash From 1f6dc2c744a38cd1de6959347003682834541605 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 22:44:13 +0530 Subject: [PATCH 370/513] test tar.gz --- release-pipelines.yml | 494 ++++++++++++++++++++---------------------- 1 file changed, 233 insertions(+), 261 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9cc145eea..63c8ca9a0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -62,8 +62,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives steps: #TODO: remove this after merging to main @@ -120,54 +118,19 @@ stages: displayName: 'Publish binaries' condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - ls -ltR $(work_dir) - displayName: 'Create required directories' - - script: | mkdir -p pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ + mv $(root_dir)/azcopy_linux_amd64 pkgDir/usr/bin/azcopy cp NOTICE.txt pkgDir/usr/bin/ displayName: 'Copy required files for packaging' @@ -195,33 +158,12 @@ stages: workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() - + - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' @@ -247,8 +189,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives steps: #TODO: remove this after merging to main @@ -304,54 +244,21 @@ stages: artifactName: 'azCopy-binaries' displayName: 'Publish binaries' condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ + mv $(root_dir)/azcopy_linux_arm64 pkgDir/usr/bin/azcopy displayName: 'Copy required files for packaging' # using fpm tool for packaging of our binary & performing post-install operations @@ -378,27 +285,6 @@ stages: workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' @@ -493,102 +379,102 @@ stages: # verbosity: 'Verbose' # alertWarningLevel: 'High' - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -608,10 +494,15 @@ stages: variables: - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives steps: - - checkout: none - + - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' @@ -651,7 +542,7 @@ stages: parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, *tar.gz' + Pattern: '*.rpm, *.deb, azcopy_linux*' DisplayName: 'ESRP CodeSigning azCopy linux' - template: setup/esrp_sign_linux.yml @@ -661,6 +552,61 @@ stages: Pattern: '*.rpm, *.deb' DisplayName: 'ESRP CodeSigning azCopy mariner' + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + ls -ltR $(work_dir) + displayName: 'Create required directories' + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* @@ -676,6 +622,42 @@ stages: md5sum $(Build.ArtifactStagingDirectory)/* md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + displayName: 'Add tar.gz to staging directory' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 @@ -1006,42 +988,42 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - job: Set_2 timeoutInMinutes: 120 @@ -1064,15 +1046,12 @@ stages: - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-temp' + artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-temp/azcopy_linux_amd64*.tar.gz' + item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - script: | - #!/bin/bash - # Define the path to the .tar.gz file and the extraction directory - # Define variables TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" @@ -1113,16 +1092,9 @@ stages: exit 1 fi - # Check if azcopy exists and is executable - if [ ! -x "./azcopy" ]; then - echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" - exit 1 - fi - # Run azcopy commands ./azcopy --version ./azcopy --help - displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-temp @@ -1828,8 +1800,8 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: From ec7ba15a4a58580018416423bff709d30397513f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 2 Jul 2024 22:44:13 +0530 Subject: [PATCH 371/513] test tar.gz --- release-pipelines.yml | 500 ++++++++++++++++++++---------------------- 1 file changed, 239 insertions(+), 261 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 9cc145eea..4a105c58e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -62,8 +62,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives steps: #TODO: remove this after merging to main @@ -120,54 +118,19 @@ stages: displayName: 'Publish binaries' condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - ls -ltR $(work_dir) - displayName: 'Create required directories' - - script: | mkdir -p pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ + mv $(root_dir)/azcopy_linux_amd64 pkgDir/usr/bin/azcopy cp NOTICE.txt pkgDir/usr/bin/ displayName: 'Copy required files for packaging' @@ -195,33 +158,12 @@ stages: workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() - + - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' @@ -247,8 +189,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives steps: #TODO: remove this after merging to main @@ -304,54 +244,21 @@ stages: artifactName: 'azCopy-binaries' displayName: 'Publish binaries' condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ + mv $(root_dir)/azcopy_linux_arm64 pkgDir/usr/bin/azcopy displayName: 'Copy required files for packaging' # using fpm tool for packaging of our binary & performing post-install operations @@ -378,27 +285,6 @@ stages: workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' @@ -493,102 +379,102 @@ stages: # verbosity: 'Verbose' # alertWarningLevel: 'High' - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -608,10 +494,15 @@ stages: variables: - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives steps: - - checkout: none - + - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' @@ -651,7 +542,7 @@ stages: parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, *tar.gz' + Pattern: '*.rpm, *.deb, azcopy_linux*' DisplayName: 'ESRP CodeSigning azCopy linux' - template: setup/esrp_sign_linux.yml @@ -661,6 +552,67 @@ stages: Pattern: '*.rpm, *.deb' DisplayName: 'ESRP CodeSigning azCopy mariner' + - script: | + export GOARCH=amd64 + export GOOS=linux + # Set the azcopy binary path + azcopyBinary="$(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64" + + # Ensure the azcopy binary has execution permissions + chmod +x "$azcopyBinary" + + # Run azcopy --version and capture the output + azcopyOutput=$($azcopyBinary --version 2>&1) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + ls -ltR $(work_dir) + displayName: 'Create required directories' + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* @@ -676,6 +628,42 @@ stages: md5sum $(Build.ArtifactStagingDirectory)/* md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + displayName: 'Add tar.gz to staging directory' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 @@ -1006,42 +994,42 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - job: Set_2 timeoutInMinutes: 120 @@ -1064,15 +1052,12 @@ stages: - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-temp' + artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-temp/azcopy_linux_amd64*.tar.gz' + item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - script: | - #!/bin/bash - # Define the path to the .tar.gz file and the extraction directory - # Define variables TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" @@ -1113,16 +1098,9 @@ stages: exit 1 fi - # Check if azcopy exists and is executable - if [ ! -x "./azcopy" ]; then - echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" - exit 1 - fi - # Run azcopy commands ./azcopy --version ./azcopy --help - displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-temp @@ -1828,8 +1806,8 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: From 48df5d8e36645d61780702abc21cbc296286f33e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 08:51:03 +0530 Subject: [PATCH 372/513] test tar.gz --- release-pipelines.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b020f9095..4a105c58e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -555,7 +555,6 @@ stages: - script: | export GOARCH=amd64 export GOOS=linux -<<<<<<< HEAD # Set the azcopy binary path azcopyBinary="$(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64" @@ -565,11 +564,6 @@ stages: # Run azcopy --version and capture the output azcopyOutput=$($azcopyBinary --version 2>&1) -======= - # Run azcopy --version and capture the output - azcopyOutput=$($(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 --version) - ->>>>>>> 1f6dc2c744a38cd1de6959347003682834541605 # Extract the version part if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') From 42cf70a4eb46d7ba447af3479a1632f3b1439bf5 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 08:52:49 +0530 Subject: [PATCH 373/513] test tar.gz --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4a105c58e..f56302c99 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -555,6 +555,7 @@ stages: - script: | export GOARCH=amd64 export GOOS=linux + # Set the azcopy binary path azcopyBinary="$(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64" From 118f4be6bad1d6d7660239c0879a139cd978f9bb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 09:10:56 +0530 Subject: [PATCH 374/513] test tar.gz --- release-pipelines.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f56302c99..f260e0ca3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -555,16 +555,11 @@ stages: - script: | export GOARCH=amd64 export GOOS=linux - - # Set the azcopy binary path - azcopyBinary="$(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64" + # Run azcopy --version and capture the output - # Ensure the azcopy binary has execution permissions - chmod +x "$azcopyBinary" + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 + azcopyOutput=$($(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 --version) - # Run azcopy --version and capture the output - azcopyOutput=$($azcopyBinary --version 2>&1) - # Extract the version part if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') From 1aecfaabf1861acf131876420e2f673d45735ab2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 09:14:22 +0530 Subject: [PATCH 375/513] test version before signing --- release-pipelines.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index f260e0ca3..b9c901504 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -538,25 +538,11 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux' - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/mariner - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - Pattern: '*.rpm, *.deb' - DisplayName: 'ESRP CodeSigning azCopy mariner' - - - script: | + - script: | export GOARCH=amd64 export GOOS=linux + # Run azcopy --version and capture the output - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 azcopyOutput=$($(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 --version) @@ -578,6 +564,20 @@ stages: fi displayName: 'Verify and extract the generated build version' + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb, azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + displayName: 'ESRP CodeSigning azCopy mariner' + - script: | linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" From e1cbb3ffd047ff5027ba3ac77a293c95c5357e9d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 09:14:52 +0530 Subject: [PATCH 376/513] test version before signing --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b9c901504..71a55623a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -538,7 +538,7 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - script: | + - script: | export GOARCH=amd64 export GOOS=linux From e33069b026a69e9d5070bec53fd0e510f96acff2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 11:06:59 +0530 Subject: [PATCH 377/513] complete test --- release-pipelines.yml | 1950 +++++++++++++++++++++-------------------- 1 file changed, 989 insertions(+), 961 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 71a55623a..067582b5d 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -62,6 +62,8 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives steps: #TODO: remove this after merging to main @@ -118,19 +120,54 @@ stages: displayName: 'Publish binaries' condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + ls -ltR $(work_dir) + displayName: 'Create required directories' + - script: | mkdir -p pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 pkgDir/usr/bin/azcopy + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ cp NOTICE.txt pkgDir/usr/bin/ displayName: 'Copy required files for packaging' @@ -158,12 +195,33 @@ stages: workingDirectory: $(root_dir) displayName: 'Make AMD64 rpm Package' + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' displayName: 'Publish Artifacts' condition: succeededOrFailed() - + - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' @@ -189,6 +247,8 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives steps: #TODO: remove this after merging to main @@ -244,21 +304,54 @@ stages: artifactName: 'azCopy-binaries' displayName: 'Publish binaries' condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" + - script: | + export GOARCH=arm64 + export GOOS=linux + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + - script: | mkdir -p pkgDir/usr/bin/ cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 pkgDir/usr/bin/azcopy + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ displayName: 'Copy required files for packaging' # using fpm tool for packaging of our binary & performing post-install operations @@ -285,6 +378,27 @@ stages: workingDirectory: $(root_dir) displayName: 'Make ARM64 rpm Package' + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to staging directory' + - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-linux-temp' @@ -297,184 +411,184 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - # displayName: 'Remove arm7 binary from staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe + displayName: 'Remove arm7 binary from staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -494,15 +608,10 @@ stages: variables: - group: AZCOPY_ESRP_SECRET - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives steps: - + - checkout: none + - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' @@ -538,37 +647,11 @@ stages: sudo apt install dotnet-sdk-3.1 -y displayName: "Update dependencies" - - script: | - export GOARCH=amd64 - export GOOS=linux - - # Run azcopy --version and capture the output - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 - azcopyOutput=$($(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' - - template: setup/esrp_sign_linux.yml parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, azcopy_linux*' + Pattern: '*.rpm, *.deb' DisplayName: 'ESRP CodeSigning azCopy linux' - template: setup/esrp_sign_linux.yml @@ -576,40 +659,9 @@ stages: FolderPath: $(Build.ArtifactStagingDirectory)/mariner ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) Pattern: '*.rpm, *.deb' - displayName: 'ESRP CodeSigning azCopy mariner' + DisplayName: 'ESRP CodeSigning azCopy mariner' - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ - ls -ltR $(work_dir) - displayName: 'Create required directories' - - # Validate signed images have md5sum changed + # Validate signed images have md5sum changed - script: | chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* @@ -624,365 +676,329 @@ stages: md5sum $(Build.ArtifactStagingDirectory)/* md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' + + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' + - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' + displayName: 'Archive Windows 64 bit AMD Build' inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' + displayName: 'Archive Windows 64 bit ARM Build' inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' continueOnError: true - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' + displayName: 'Archive Windows 32 bit Build' inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' continueOnError: true - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - displayName: 'Add tar.gz to staging directory' + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-linux-signed' + artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' - - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # displayName: 'List Artifacts' - - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - - # - script: | - # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir -p $windows_amd64 - # mkdir -p $windows_arm64 - # mkdir -p $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p unzipped_amd64 - # mkdir -p unzipped_arm64 - # unzip -o azcopy*amd64*.zip -d unzipped_amd64 - # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - - # mv $amd64FolderName/azcopy azcopy_darwin_amd64 - # mv $arm64FolderName/azcopy azcopy_darwin_arm64 - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + + mv $amd64FolderName/azcopy azcopy_darwin_amd64 + mv $arm64FolderName/azcopy azcopy_darwin_arm64 + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Mac Binary' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Binaries-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_linux.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - # Pattern: 'azcopy_linux*' - # DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: 'azcopy_linux*' + DisplayName: 'ESRP CodeSigning azCopy linux binaries' + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' # SignArtifacts end here @@ -990,42 +1006,42 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - job: Set_2 timeoutInMinutes: 120 @@ -1053,7 +1069,10 @@ stages: item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - script: | + #!/bin/bash + # Define the path to the .tar.gz file and the extraction directory + # Define variables TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" @@ -1094,362 +1113,368 @@ stages: exit 1 fi + # Check if azcopy exists and is executable + if [ ! -x "./azcopy" ]; then + echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" + exit 1 + fi + # Run azcopy commands ./azcopy --version ./azcopy --help + + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install -y + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-temp - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22-arm64' - # AgentName: "blobfuse-ubn22-arm64" - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install -y - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - # - script: | - # sudo dpkg --info azcopy-*arm64.deb - # sudo dpkg -i azcopy-*arm64.deb - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' - - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Arm64: - # agentName: "blobfuse-rhel9-arm64" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9-arm64' + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*.arm64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*.arm64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' - # - job: Set_5 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Rhel_Amd64: - # agentName: "blobfuse-rhel9" - # vmImage: 'RHEL-9.0' - # container: 'test-cnt-rhel-9' + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' - # pool: - # name: "blobfuse-rhel-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # steps: - # - checkout: none - # - script: | - # sudo yum update -y - # sudo yum install git -y - # displayName: 'Install Git' - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - # sudo rpm -qip azcopy-*x86_64.rpm - # sudo yum groupinstall "Development Tools" -y - # sudo rpm -i azcopy-*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' - - # - job: Set_6 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mariner2: - # agentName: "blobfuse-mariner2" - # DistroVer: "Mariner2" - # Description: "CBL-Mariner2 Linux" - # container: "test-cnt-mari-2" - - # pool: - # name: "blobfuse-mariner-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # steps: - # - checkout: none - - # - script: | - # sudo tdnf update -y - # sudo tdnf install git -y - # displayName: 'Install dependencies' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_6 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # sudo rpm -qip azcopy*x86_64.rpm - # sudo rpm -i azcopy*x86_64.rpm - # displayName: 'Install Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help command' - - # - job: Set_7 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - # - script: | - # # Install Homebrew if not already installed - # if ! command -v brew &> /dev/null - # then - # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - # echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - # eval "$(/opt/homebrew/bin/brew shellenv)" - # fi - - # # Define variables - # ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Create extraction directory - # mkdir -p $EXTRACT_DIR - - # # Extract the zip file - # unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # # Find the directory matching the pattern - # matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # # Run the azcopy version command - # ./azcopy --version - # ./azcopy --help - # displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_7 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - # - job: Set_8 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_8 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-signed' - # downloadPath: $(Build.ArtifactStagingDirectory) - # itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - # - task: PowerShell@2 - # inputs: - # targetType: 'inline' - # script: | - # $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - # $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - # # Find the zip file matching the pattern - # $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - # if ($null -eq $zipFile) { - # Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - # exit 1 - # } - - # $zipFilePath = $zipFile.FullName - - # # Create extraction directory - # New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # # Extract the zip file - # try { - # Add-Type -AssemblyName System.IO.Compression.FileSystem - # [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - # } catch { - # Write-Error "Failed to extract zip file: $_" - # exit 1 - # } - - # # Change to the directory containing azcopy - # Set-Location -Path $extractDir - - # # Find directories matching the pattern - # $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # # Check if there is exactly one matching directory - # if ($matchingDirs.Count -eq 0) { - # Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } elseif ($matchingDirs.Count -gt 1) { - # Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - # exit 1 - # } - - # # Check if the azcopy executable exists in the matching directory - # $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" - # if (-Not (Test-Path -Path $azcopyPath)) { - # Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - # exit 1 - # } else { - # Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # # Ensure the azcopy executable has the correct permissions - # icacls $azcopyPath - - # # Run azcopy --version and capture output - # try { - # $versionOutput = & $azcopyPath --version 2>&1 - # Write-Output "azcopy version output: $versionOutput" - # } catch { - # Write-Error "Failed to run azcopy --version: $_" - # } - - # # Run azcopy --help and capture output - # try { - # $helpOutput = & $azcopyPath --help 2>&1 - # Write-Output "azcopy help output: $helpOutput" - # } catch { - # Write-Error "Failed to run azcopy --help: $_" - # } - # } - # displayName: 'Extract Files and Run Version and Help Command on Windows' - - # # TestArtifacts ends here + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } + } + displayName: 'Extract Files and Run Version and Help Command on Windows' + # TestArtifacts ends here - stage: ReleaseArtifacts dependsOn: TestArtifacts @@ -1520,7 +1545,10 @@ stages: # List the contents of the resulting zip file for verification unzip -l $(Build.ArtifactStagingDirectory)/drop.zip + + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . displayName: 'Archive all binaries to drop.zip' + workingDirectory: '$(Build.ArtifactStagingDirectory)' - task: Bash@3 displayName: 'Install Azure CLI' @@ -1802,8 +1830,8 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: From 4f39c582ded144b92350eba1dcdce93757268268 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 12:46:46 +0530 Subject: [PATCH 378/513] complete test --- packages.csv | 2 -- release-pipelines.yml | 82 +++++++++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/packages.csv b/packages.csv index a6a984885..054d11f89 100644 --- a/packages.csv +++ b/packages.csv @@ -24,5 +24,3 @@ CentOS-8.0,azcopyAmdRpm,microsoft-centos8-prod-yum, SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, -Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-preview-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerAarchRpm,cbl-mariner-2.0-preview-Microsoft-aarch64-yum, diff --git a/release-pipelines.yml b/release-pipelines.yml index 067582b5d..c3549b83b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -906,10 +906,30 @@ stages: # Use find to retrieve the folder name amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" - mv $amd64FolderName/azcopy azcopy_darwin_amd64 - mv $arm64FolderName/azcopy azcopy_darwin_arm64 + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip @@ -1546,7 +1566,7 @@ stages: # List the contents of the resulting zip file for verification unzip -l $(Build.ArtifactStagingDirectory)/drop.zip - cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . + cp -r $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . displayName: 'Archive all binaries to drop.zip' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -1783,49 +1803,41 @@ stages: is_preview="false" echo "##vso[task.setvariable variable=is_preview]$is_preview" - if [[ $marinerAmdRpmFile == *"preview"* ]]; then is_preview="true" echo "##vso[task.setvariable variable=is_preview]$is_preview" - echo "Package $marinerAmdRpmFile contains 'preview'." - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +5 ../packages.csv) - else - echo "Package $package_name1 does not contain 'preview'." - while IFS=, read -r distro fuseArchType repoName releaseName; do - echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName - done < <(tail -n +3 ../packages.csv | head -n -2) - fi + while IFS=, read -r distro archType repoName releaseName; do + + # If the package is preview, publish to mariner preview package + if [[ $distro == *"Mariner-2.0"* ]]; then + if [ "$(is_preview)" = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + + echo "Uploading packages for $distro" + echo "Repo Name: $repoName" + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - if [ "$(is_preview)" = "true" ]; then - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then + while IFS=, read -r distro archType repoName releaseName; do + if [[ $ArchType == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + if [[ $distro == *"Mariner-2.0"* ]]; then + if [ "$(is_preview)" = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + echo "Repo Name: $repoName" + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName fi - done < <(tail -n +5 ../packages.csv) - else - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv | head -n -2) - fi - + done < <(tail -n +3 ../packages.csv) displayName: 'Publish the repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ From fd9830f5b4ba6f0ee8baab189852fd99b2bca4ff Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 12:49:00 +0530 Subject: [PATCH 379/513] complete test --- release-pipelines.yml | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c3549b83b..65e3bc0e3 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1760,7 +1760,7 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1772,34 +1772,31 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - echo "Azcopy AMD DEB $azcopyAmdDeb" - echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + #azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + #echo "Azcopy AMD DEB ID: $azcopyAmdDeb" azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy AMD RPM $azcopyAmdRpmFile" - echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - echo "Azcopy ARM DEB $azcopyArmDeb" - echo "Azcopy ARM DEB ID: $azcopyArmDeb" + # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + # echo "Azcopy ARM DEB ID: $azcopyArmDeb" azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - echo "Azcopy ARM RPM $azcopyArmRpmFile" - echo "Azcopy ARM RPM ID: $azcopyArmRpm" + # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + # echo "Azcopy ARM RPM $azcopyArmRpmFile" + # echo "Azcopy ARM RPM ID: $azcopyArmRpm" # Find the Mariner RPM file and upload marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" is_preview="false" echo "##vso[task.setvariable variable=is_preview]$is_preview" From 5c0d283c59f5e3da0d1ef80d012d6aadaadd2546 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 13:12:00 +0530 Subject: [PATCH 380/513] complete test --- release-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 65e3bc0e3..a45103101 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -921,6 +921,8 @@ stages: exit 1 fi + cd $(Build.ArtifactStagingDirectory) + # Check if there is exactly one matching directory for arm64 if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then echo "Found matching ARM64 directory: $arm64FolderName" From 7dbfe1ed1edb228252315c6ebf39e00b383c8264 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 13:20:38 +0530 Subject: [PATCH 381/513] incorporated review comments --- go_installer.sh | 2 +- packages.csv | 8 +- release-pipeline.yml | 587 +++++++++++++++++++++------------- setup/releaseVersionUpdate.py | 36 +-- setup/test_artifacts.yml | 5 +- 5 files changed, 398 insertions(+), 240 deletions(-) diff --git a/go_installer.sh b/go_installer.sh index 932951471..d1f569fd1 100644 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.21.8" +version=$2 arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ];then diff --git a/packages.csv b/packages.csv index 0676436d0..054d11f89 100644 --- a/packages.csv +++ b/packages.csv @@ -1,5 +1,7 @@ # Do not remove these comments -# Format of the file : ,,, +# Format of the file : ,,, +Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, +Mariner-2.0-aarch64,marinerAarchRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, Ubuntu-18.04,azcopyAmdDeb,microsoft-ubuntu-bionic-prod-apt,bionic Ubuntu-20.04,azcopyAmdDeb,microsoft-ubuntu-focal-prod-apt,focal Ubuntu-22.04,azcopyAmdDeb,microsoft-ubuntu-jammy-prod-apt,jammy @@ -20,7 +22,5 @@ RHEL-9.0,azcopyArmRpm,microsoft-rhel9.0-prod-yum, CentOS-7.0,azcopyAmdRpm,microsoft-centos7-prod-yum, CentOS-8.0,azcopyAmdRpm,microsoft-centos8-prod-yum, SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, -Mariner-2.0-x86_64,marinerAmdRpm,cbl-mariner-2.0-prod-Microsoft-x86_64-yum, -Mariner-2.0-aarch64,marinerArmRpm,cbl-mariner-2.0-prod-Microsoft-aarch64-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, -Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, \ No newline at end of file +Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, diff --git a/release-pipeline.yml b/release-pipeline.yml index fd6f4d484..a45103101 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -64,10 +64,6 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - name: archives value: $(root_dir)/archives - - name: linux_amd64 - value: $(work_dir)/linux_amd64 - - name: linux_se_amd64 - value: $(work_dir)/linux_se_amd64 steps: #TODO: remove this after merging to main @@ -81,7 +77,7 @@ stages: - task: ShellScript@2 inputs: scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" displayName: "Installing Go tools" - script: | @@ -106,26 +102,17 @@ stages: displayName: "GLIBC Version" - script: | - mkdir $(linux_amd64) - mkdir $(linux_se_amd64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(linux_amd64)/azcopy_linux_amd64" + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" displayName: 'Generate Linux AMD64' - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_amd64)/azcopy_linux_se_amd64" + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" displayName: 'Generate Linux AMD64 SE Integration' - script: | - sudo ls -lRt $(linux_amd64)/ - sudo ls -lRt $(linux_se_amd64)/ - cp $(linux_se_amd64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp $(linux_amd64)/azcopy_linux_amd64 $(Build.ArtifactStagingDirectory) + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) displayName: 'List the binaries and add to staging directory' - workingDirectory: $(root_dir) - task: PublishBuildArtifacts@1 inputs: @@ -137,30 +124,13 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(linux_amd64)/azcopy_linux_amd64 $(linux_amd64)/azcopy - mv $(linux_se_amd64)/azcopy_linux_se_amd64 $(linux_se_amd64)/azcopy - cp $(linux_amd64)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64)/ - cp NOTICE.txt $(linux_se_amd64)/ - displayName: 'Copy required files for packaging' - - - script: | - sudo ls -lRt pkgDir/usr/bin/ - displayName: 'List the files' - - script: | export GOARCH=amd64 export GOOS=linux - $(linux_amd64)/azcopy --version - azcopyOutput=$($(linux_amd64)/azcopy --version) - # Run azcopy --version and capture the output - azcopyOutput=$($(linux_amd64)/azcopy --version) + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - # Extract the version part using grep and sed + # Extract the version part if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') if [[ -n "$version" ]]; then @@ -177,11 +147,34 @@ stages: exit 1 fi displayName: 'Verify and extract the generated build version' - + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + ls -ltR $(work_dir) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + cp NOTICE.txt pkgDir/usr/bin/ + displayName: 'Copy required files for packaging' + # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -205,7 +198,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Partner(SE) Linux Build' inputs: - rootFolderOrFile: '$(linux_se_amd64)' + rootFolderOrFile: '$(linux_se_amd64_dir)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' continueOnError: true @@ -213,7 +206,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Amd Linux Build' inputs: - rootFolderOrFile: '$(linux_amd64)' + rootFolderOrFile: '$(linux_amd64_dir)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' continueOnError: true @@ -256,10 +249,6 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - name: archives value: $(root_dir)/archives - - name: linux_arm64 - value: $(work_dir)/linux_arm64 - - name: linux_se_arm64 - value: $(work_dir)/linux_se_arm64 steps: #TODO: remove this after merging to main @@ -273,7 +262,7 @@ stages: - task: ShellScript@2 inputs: scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" displayName: "Installing Go tools" - script: | @@ -298,26 +287,17 @@ stages: displayName: "GLIBC Version" - script: | - mkdir $(linux_arm64) - mkdir $(linux_se_arm64) - mkdir $(archives) - displayName: "Create required directories" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(linux_arm64)/azcopy_linux_arm64" + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" displayName: 'Generate Linux ARM64' - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(linux_se_arm64)/azcopy_linux_se_arm64" + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" displayName: 'Generate Linux ARM64 SE Integration' - script: | - sudo ls -lRt $(linux_arm64)/ - sudo ls -lRt $(linux_se_arm64)/ - cp -r $(linux_se_arm64)/azcopy_linux_se* $(Build.ArtifactStagingDirectory) - cp -r $(linux_arm64)/azcopy_linux_arm64 $(Build.ArtifactStagingDirectory) + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) displayName: 'List the binaries and add to staging directory' - workingDirectory: $(root_dir) - task: PublishBuildArtifacts@1 inputs: @@ -329,22 +309,12 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - cp NOTICE.txt $(linux_se_arm64)/ - cp NOTICE.txt $(linux_arm64)/ - mv $(linux_arm64)/azcopy_linux_arm64 $(linux_arm64)/azcopy - cp $(linux_arm64)/azcopy pkgDir/usr/bin/ - displayName: 'Copy required files' - - script: | export GOARCH=arm64 export GOOS=linux - $(linux_arm64)/azcopy --version - azcopyOutput=$($(linux_arm64)/azcopy --version) + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - # Extract the version part using grep and sed + # Extract the version part if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') if [[ -n "$version" ]]; then @@ -362,10 +332,32 @@ stages: fi displayName: 'Verify and extract the generated build version' + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -389,7 +381,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Standard Linux Build' inputs: - rootFolderOrFile: '$(linux_arm64)' + rootFolderOrFile: '$(linux_arm64_dir)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' continueOnError: true @@ -397,7 +389,7 @@ stages: - task: ArchiveFiles@2 displayName: 'Archive Partner(SE) Linux Build' inputs: - rootFolderOrFile: '$(linux_se_arm64)' + rootFolderOrFile: '$(linux_se_arm64_dir)' archiveType: tar archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' continueOnError: true @@ -405,7 +397,7 @@ stages: - script: | cp -r $(archives)/* $(Build.ArtifactStagingDirectory) workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' + displayName: 'Add tar.gz to staging directory' - task: PublishBuildArtifacts@1 inputs: @@ -516,10 +508,6 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: darwin_amd_dir - value: '$(work_dir)/azcopy_darwin_amd64' - - name: darwin_arm_dir - value: '$(work_dir)/azcopy_darwin_arm64' - name: archives value: $(root_dir)/archives @@ -537,48 +525,39 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | - mkdir $(darwin_amd_dir) - mkdir $(darwin_arm_dir) - mkdir $(archives) - displayName: 'Create required directories' - - - script: | - CGO_ENABLED=1 go build -o "$(darwin_amd_dir)/azcopy_darwin_amd64" + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" displayName: 'Generate MacOS Build with AMD64' - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(darwin_arm_dir)/azcopy_darwin_arm64" + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - script: | - cp $(darwin_amd_dir)/azcopy_darwin_amd64 $(Build.ArtifactStagingDirectory) - cp $(darwin_arm_dir)/azcopy_darwin_arm64 $(Build.ArtifactStagingDirectory) - displayName: "Copy binaries to staging directory" - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - displayName: "Remove binaries from staging directory" - task: Bash@3 displayName: 'Extract AZCopy version' inputs: targetType: 'inline' script: | - azcopy_version=$($(darwin_amd_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + - script: | cp NOTICE.txt $(darwin_amd_dir) cp NOTICE.txt $(darwin_arm_dir) - mv $(darwin_amd_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(darwin_arm_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy NOTICE file' + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' - task: ArchiveFiles@2 displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' @@ -595,8 +574,7 @@ stages: continueOnError: true - script: | - cp $(archives)/azcopy_darwin_amd64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_darwin_arm64_$(azcopy_version).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - task: PublishBuildArtifacts@1 @@ -673,7 +651,7 @@ stages: parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb, *tar.gz' + Pattern: '*.rpm, *.deb' DisplayName: 'ESRP CodeSigning azCopy linux' - template: setup/esrp_sign_linux.yml @@ -690,7 +668,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp displayName: 'Make Artifacts executable' - script: | @@ -721,12 +699,6 @@ stages: - group: AZCOPY_ESRP_SECRET - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: windows_amd64 - value: '$(work_dir)/azcopy_windows_amd64' - - name: windows_arm64 - value: '$(work_dir)/azcopy_windows_arm64' - - name: windows_386 - value: '$(work_dir)/azcopy_windows_386' - name: archives value: '$(work_dir)/archives' @@ -752,13 +724,6 @@ stages: sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* displayName: 'List Artifacts' - - - script: | - mkdir -p $(windows_amd64) - mkdir -p $(windows_arm64) - mkdir -p $(windows_386) - mkdir -p $(archives) - displayName: 'Create required directories' - script: | sudo apt-get clean @@ -776,7 +741,7 @@ stages: parameters: FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: '*.exe' + Pattern: 'azcopy_windows*' # Validate signed images have md5sum changed - script: | @@ -799,15 +764,28 @@ stages: displayName: 'Extract the generated build version' - script: | - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64) - cp $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386) + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe cp NOTICE.txt $(windows_amd64) cp NOTICE.txt $(windows_arm64) cp NOTICE.txt $(windows_386) - mv $(windows_amd64)/azcopy_windows_amd64 $(windows_amd64)/azcopy - mv $(windows_arm64)/azcopy_windows_arm64 $(windows_arm64)/azcopy - mv $(windows_386)/azcopy_windows_386 $(windows_386)/azcopy zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . @@ -835,9 +813,7 @@ stages: continueOnError: true - script: | - cp $(archives)/azcopy_windows_amd64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_arm64_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) - cp $(archives)/azcopy_windows_386_$(AZCOPY_VERSION).zip $(Build.ArtifactStagingDirectory) + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) displayName: 'Copy zip to staging directory' - script: | @@ -923,16 +899,45 @@ stages: displayName: 'Publish Signed Artifacts' - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_amd64 - mkdir -p $(Build.ArtifactStagingDirectory)/unzipped_arm64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*amd64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_amd64 - unzip -o $(Build.ArtifactStagingDirectory)/azcopy*arm64*.zip -d $(Build.ArtifactStagingDirectory)/unzipped_arm64 - mv $(Build.ArtifactStagingDirectory)/unzipped_amd64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - mv $(Build.ArtifactStagingDirectory)/unzipped_arm64/azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy signed mac binary to staging directory' + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' # Push signed images to artifact directory - task: PublishBuildArtifacts@1 @@ -995,7 +1000,7 @@ stages: parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: '*.exe' + Pattern: 'azcopy_windows*' # Validate signed images have md5sum changed - script: | @@ -1043,6 +1048,7 @@ stages: - template: setup/test_artifacts.yml parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' @@ -1058,8 +1064,91 @@ stages: azcopy --version azcopy --help displayName: 'Check Version and Help' - + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + + - script: | + #!/bin/bash + + # Define the path to the .tar.gz file and the extraction directory + # Define variables + TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Check if the file exists and is a regular file + if [ ! -f "$TAR_GZ_FILE" ]; then + echo "Error: File $TAR_GZ_FILE does not exist." + exit 1 + fi + + # Verify the file type + file_type=$(file -b "$TAR_GZ_FILE") + echo "File type: $file_type" + + if [[ "$file_type" != *"gzip compressed data"* ]]; then + echo "Error: The file is not a valid .tar.gz file" + exit 1 + fi + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Check if azcopy exists and is executable + if [ ! -x "./azcopy" ]; then + echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 timeoutInMinutes: 120 strategy: matrix: @@ -1076,12 +1165,13 @@ stages: - checkout: none - script: | sudo apt update - sudo apt --fix-broken install + sudo apt --fix-broken install -y ldd --version displayName: "GLIBC Version" - template: setup/test_artifacts.yml parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' @@ -1097,7 +1187,7 @@ stages: azcopy --help displayName: 'Check Version and Help' - - job: Set_3 + - job: Set_4 timeoutInMinutes: 120 strategy: matrix: @@ -1120,6 +1210,7 @@ stages: - template: setup/test_artifacts.yml parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' @@ -1135,9 +1226,9 @@ stages: - script: | azcopy --version azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help command' - - job: Set_4 + - job: Set_5 timeoutInMinutes: 120 strategy: matrix: @@ -1160,6 +1251,7 @@ stages: - template: setup/test_artifacts.yml parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' @@ -1180,9 +1272,9 @@ stages: - script: | azcopy --version azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help command' - - job: Set_5 + - job: Set_6 timeoutInMinutes: 120 strategy: matrix: @@ -1199,16 +1291,18 @@ stages: steps: - checkout: none + - script: | sudo tdnf update -y sudo tdnf install git -y - displayName: 'Install Git' - + displayName: 'Install dependencies' + - template: setup/test_artifacts.yml parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy*x86_64.rpm' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' # get glibc version with which build is done - script: | @@ -1224,9 +1318,9 @@ stages: - script: | azcopy --version azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help command' - - job: Set_6 + - job: Set_7 timeoutInMinutes: 360 strategy: matrix: @@ -1281,14 +1375,23 @@ stages: # Extract the zip file unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - # Change to the directory containing azcopy - cd $EXTRACT_DIR/azcopy_darwin_amd64 + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi # Run the azcopy version command ./azcopy --version - displayName: 'Install Zip, Extract Files, and Run Version Command' + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_7 + - job: Set_8 timeoutInMinutes: 120 strategy: matrix: @@ -1354,18 +1457,47 @@ stages: # Change to the directory containing azcopy Set-Location -Path $extractDir - # Verify that azcopy.exe exists - if (-Not (Test-Path -Path ".\azcopy_windows_amd64\azcopy")) { - Write-Error "azcopy.exe not found in extraction directory: $extractDir" - exit 1 + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 } - # Run the azcopy version command - .\azcopy_windows_amd64\azcopy --version - displayName: 'Extract Files and Run Version Command on Windows' + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } + } + displayName: 'Extract Files and Run Version and Help Command on Windows' + # TestArtifacts ends here - # TestArtifacts ends here - - stage: ReleaseArtifacts dependsOn: TestArtifacts condition: succeeded('TestArtifacts') @@ -1415,21 +1547,30 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + cp NOTICE.txt azCopy-binaries-signed/ sudo apt-get install -y zip - zip -r $(Build.ArtifactStagingDirectory)/drop.zip . cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: $(Build.ArtifactStagingDirectory) + workingDirectory: '$(Build.ArtifactStagingDirectory)' - - task: ArchiveFiles@2 + #TODO: check once + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" + ls -l + + # Create the zip archive with all files inside azCopy-binaries-signed + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + + # List the contents of the resulting zip file for verification + unzip -l $(Build.ArtifactStagingDirectory)/drop.zip + + cp -r $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . displayName: 'Archive all binaries to drop.zip' - inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/azCopy-binaries-signed' - archiveFile: '$(Build.ArtifactStagingDirectory)/drop.zip' - continueOnError: true + workingDirectory: '$(Build.ArtifactStagingDirectory)' - task: Bash@3 displayName: 'Install Azure CLI' @@ -1449,8 +1590,6 @@ stages: raw_version_output=$(azcopy --version) azcopy_version=$(echo $raw_version_output | awk '{print $3}') - ls -ltR $(Build.ArtifactStagingDirectory) - today=$(date +"%Y%m%d") if [ $(Test_Mode) = "True" ]; then @@ -1509,6 +1648,7 @@ stages: sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip displayName: 'List Artifacts' @@ -1595,7 +1735,6 @@ stages: fi displayName: 'Test PMC installation' - # download artifacts that need to be published - task: DownloadBuildArtifacts@0 displayName: 'Download Signed Artifacts' inputs: @@ -1610,7 +1749,7 @@ stages: marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - ls -lRt + sudo ls -lRt mv azcopy*.rpm .. cd .. rm -r mariner/ @@ -1623,11 +1762,11 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) - # replace org name while merging to main + # TODO: replace org name while merging to main - script: | wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv cat ./packages.csv @@ -1635,57 +1774,75 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - # Find the Mariner RPM file and upload - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - - # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - marinerArmRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerArmRpmFile) - echo "Azcopy mariner ARM RPM ID: $marinerArmRpm" - - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + #azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + #echo "Azcopy AMD DEB ID: $azcopyAmdDeb" azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy mariner AMD RPM ID: $azcopyAmdRpm" + # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - echo "Azcopy ARM DEB ID: $azcopyArmDeb" + # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + # echo "Azcopy ARM DEB ID: $azcopyArmDeb" - azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm|') - azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - echo "Azcopy ARM RPM ID: $azcopyArmRpm" + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') + # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + # echo "Azcopy ARM RPM $azcopyArmRpmFile" + # echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + + is_preview="false" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + is_preview="true" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + + while IFS=, read -r distro archType repoName releaseName; do + + # If the package is preview, publish to mariner preview package + if [[ $distro == *"Mariner-2.0"* ]]; then + if [ "$(is_preview)" = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi - while IFS=, read -r distro fuseArchType repoName releaseName; do echo "Uploading packages for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!fuseArchType} $repoName $releaseName + echo "Repo Name: $repoName" + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro fuseArchType repoName releaseName; do - if [[ $fuseArchType == *"Arm"* ]] - then - echo "Skipping for ARM type on $distro" - else - echo "Publishing for $distro" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv) + while IFS=, read -r distro archType repoName releaseName; do + if [[ $ArchType == *"Arm"* ]]; then + echo "Skipping for ARM type on $distro" + else + if [[ $distro == *"Mariner-2.0"* ]]; then + if [ "$(is_preview)" = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + echo "Repo Name: $repoName" + #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) displayName: 'Publish the repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') + # dependsOn: ReleaseArtifacts + # condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index b4ff2d202..f994cdfce 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -13,35 +13,33 @@ print('Incorrect Release Version') sys.exit(1) +# Split the SAS URL to get the container URL and SAS token containerUrl = sasUrl.split('?')[0] sasToken = sasUrl.split('?')[1] -# delete latest version file in the container -deleteUrl = containerUrl + '/' + 'latest_version.txt' + '?' + sasToken -resp = requests.delete(deleteUrl) -sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('Deleted last release file') - -# Create a file and write data to it +# Create a file and write the release version to it file_name = 'latest_version.txt' -# Open the file in write mode with open(file_name, 'w') as file: file.write(releaseVersion) print(f'Data written to {file_name}') -# Create the BlobServiceClient using the SAS URL -blob_service_client = BlobServiceClient(account_url=sasUrl.split('?')[0], credential=sasUrl.split('?')[1]) - -# Get the container name from the SAS URL -container_name = sasUrl.split('?')[0].split('/')[-1] - -# Get a client to interact with the container -container_client = blob_service_client.get_container_client(container_name) +# Get the full URL to upload the file +putUrl = containerUrl + '/' + file_name + '?' + sasToken -# Upload the file to the container +# Upload the file using a PUT request with open(file_name, 'rb') as data: - container_client.upload_blob(name=file_name, data=data, overwrite=True) - -print(f"File '{file_name}' successfully uploaded to '{container_name}' container.") + headers = { + 'x-ms-blob-type': 'BlockBlob', + 'Content-Length': str(os.path.getsize(file_name)) + } + resp = requests.put(putUrl, data=data, headers=headers) + +# Check if the request was successful +if resp.status_code < 200 or resp.status_code > 202: + print(f"Failed to upload file: {resp.status_code}, {resp.text}") + sys.exit(1) +else: + print('File successfully uploaded') # Clean up the local file os.remove(file_name) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index f09b47044..70396f0ed 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -1,4 +1,6 @@ parameters: + - name: go_version + type: string - name: artifact_name type: string - name: download_path @@ -7,6 +9,7 @@ parameters: type: string steps: + #TODO: Change the org name after merging to main - script: | git clone https://github.com/dphulkar-msft/azure-storage-azcopy displayName: 'Checkout Code' @@ -20,7 +23,7 @@ steps: - task: ShellScript@2 inputs: scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" - args: "$(System.DefaultWorkingDirectory)/" + args: "$(System.DefaultWorkingDirectory)/ ${{ parameters.go_version }}" displayName: "GoTool Custom Setup" - task: DownloadBuildArtifacts@0 From c407efc31259b9e3caf64aeb2d72774a3a6121ed Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 13:29:40 +0530 Subject: [PATCH 382/513] incorporated review comments --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index a45103101..85ff97cdc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1805,6 +1805,7 @@ stages: if [[ $marinerAmdRpmFile == *"preview"* ]]; then is_preview="true" echo "##vso[task.setvariable variable=is_preview]$is_preview" + fi while IFS=, read -r distro archType repoName releaseName; do @@ -1819,7 +1820,6 @@ stages: echo "Repo Name: $repoName" #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) - displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ From 5b014a54b53a7a0174192299f47a2afe39643736 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 15:19:37 +0530 Subject: [PATCH 383/513] incorporated review comments --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 85ff97cdc..4382e56e0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1811,7 +1811,7 @@ stages: # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then - if [ "$(is_preview)" = "true" ]; then + if [ $is_preview = "true" ]; then repoName=$(echo $repoName | sed 's/prod/preview/') fi fi @@ -1841,8 +1841,8 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: From 81c72923ea173ef940a69b1c195dc9731e04be24 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 15:41:54 +0530 Subject: [PATCH 384/513] incorporated review comments --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4382e56e0..298893fab 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1554,7 +1554,6 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: '$(Build.ArtifactStagingDirectory)' #TODO: check once - script: | @@ -1564,11 +1563,12 @@ stages: # Create the zip archive with all files inside azCopy-binaries-signed zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - + cd $(Build.ArtifactStagingDirectory) + # List the contents of the resulting zip file for verification unzip -l $(Build.ArtifactStagingDirectory)/drop.zip - cp -r $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . displayName: 'Archive all binaries to drop.zip' workingDirectory: '$(Build.ArtifactStagingDirectory)' From 5e13b19d7c04b86dab13cde08e019f417bc816d5 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 15:44:59 +0530 Subject: [PATCH 385/513] incorporated review comments --- release-pipeline.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index a45103101..f38e6d2b6 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1554,7 +1554,6 @@ stages: sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - workingDirectory: '$(Build.ArtifactStagingDirectory)' #TODO: check once - script: | @@ -1564,11 +1563,12 @@ stages: # Create the zip archive with all files inside azCopy-binaries-signed zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - + cd $(Build.ArtifactStagingDirectory) + # List the contents of the resulting zip file for verification unzip -l $(Build.ArtifactStagingDirectory)/drop.zip - cp -r $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . displayName: 'Archive all binaries to drop.zip' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -1805,12 +1805,13 @@ stages: if [[ $marinerAmdRpmFile == *"preview"* ]]; then is_preview="true" echo "##vso[task.setvariable variable=is_preview]$is_preview" + fi while IFS=, read -r distro archType repoName releaseName; do # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then - if [ "$(is_preview)" = "true" ]; then + if [ $is_preview = "true" ]; then repoName=$(echo $repoName | sed 's/prod/preview/') fi fi @@ -1819,7 +1820,6 @@ stages: echo "Repo Name: $repoName" #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) - displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ @@ -1841,8 +1841,8 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - stage: UpdateLatestVersion - # dependsOn: ReleaseArtifacts - # condition: succeeded('ReleaseArtifacts') + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') jobs: - job: UpdateVersion pool: @@ -1888,4 +1888,3 @@ stages: python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) - From e4ba9bb0e3388bdc1f91423060d83707172d1084 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 15:51:05 +0530 Subject: [PATCH 386/513] incorporated review comments --- release-pipeline.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index f38e6d2b6..46cdc5c41 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1762,7 +1762,7 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1774,31 +1774,31 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - #azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - #echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - # echo "Azcopy ARM DEB ID: $azcopyArmDeb" + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - # echo "Azcopy ARM RPM $azcopyArmRpmFile" - # echo "Azcopy ARM RPM ID: $azcopyArmRpm" + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" + echo "Azcopy ARM RPM ID: $azcopyArmRpm" # Find the Mariner RPM file and upload marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" is_preview="false" echo "##vso[task.setvariable variable=is_preview]$is_preview" @@ -1807,7 +1807,7 @@ stages: echo "##vso[task.setvariable variable=is_preview]$is_preview" fi - while IFS=, read -r distro archType repoName releaseName; do + while IFS=, read -r distro archetype repoName releaseName; do # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1818,14 +1818,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro archType repoName releaseName; do - if [[ $ArchType == *"Arm"* ]]; then + while IFS=, read -r distro archetype repoName releaseName; do + if [[ $archetype == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1834,7 +1834,7 @@ stages: fi fi echo "Repo Name: $repoName" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName fi done < <(tail -n +3 ../packages.csv) displayName: 'Publish the repository' From f58681b22318b5d8bc484939d93154d27de729bc Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 19:17:54 +0530 Subject: [PATCH 387/513] complete test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 298893fab..5e1d02111 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1547,7 +1547,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - cp NOTICE.txt azCopy-binaries-signed/ + cp NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-linux-signed sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb From b67628d662e66d213a7aeb99408e586877a87e96 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 20:07:24 +0530 Subject: [PATCH 388/513] complete test --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5e1d02111..492076917 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1547,7 +1547,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - cp NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb @@ -1555,8 +1555,8 @@ stages: sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - #TODO: check once - script: | + cp NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" ls -l From 6aa5664bef202ea3976aade27ca81bdb740d2cff Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 22:09:47 +0530 Subject: [PATCH 389/513] complete test --- release-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 492076917..e320c8875 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -873,11 +873,11 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' # Validate signed images have md5sum changed - script: | From 624c446cd031bdfe7b355d9829026317d1e81e3e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 08:29:12 +0530 Subject: [PATCH 390/513] complete test --- release-pipelines.yml | 52 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index e320c8875..57f24614f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -471,16 +471,6 @@ stages: GOOS: windows CGO_ENABLED: 0 - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - displayName: 'Remove arm7 binary from staging directory' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-windows-temp' @@ -743,11 +733,27 @@ stages: ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) Pattern: 'azcopy_windows*' + - script: | + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries-signed' + displayName: 'Publish Signed Artifacts' + # Validate signed images have md5sum changed - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' - script: | chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 @@ -873,11 +879,11 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' # Validate signed images have md5sum changed - script: | @@ -989,13 +995,6 @@ stages: sudo apt install dotnet-sdk-6.0 -y displayName: "Update dependencies" - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - template: setup/esrp_sign_windows.yml parameters: FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' @@ -1519,6 +1518,9 @@ stages: value: 'azcopyvnextrelease' - name: Test_Mode value: ${{ parameters.test_mode }} + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + steps: @@ -1556,7 +1558,7 @@ stages: displayName: 'Create drop.zip file and install azcopy deb package' - script: | - cp NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed + cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" ls -l From 862dd0f4346e6df1c71c4d95991c3e75680d9193 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 08:43:43 +0530 Subject: [PATCH 391/513] test linux binaries --- release-pipelines.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 57f24614f..b6bd0c46b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1045,6 +1045,19 @@ stages: ldd --version displayName: "GLIBC Version" + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-binaries-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azcopy-linux*' + + - script: | + azcopy_linux_amd64 --version + azcopy_linux_se_amd64 --version + displayNameName: 'Test signed linux binaries' + workingDirectory: '$(System.DefaultWorkingDirectory)' + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1168,6 +1181,19 @@ stages: ldd --version displayName: "GLIBC Version" + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-binaries-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azcopy-linux*' + + - script: | + azcopy_linux_arm64 --version + azcopy_linux_se_arm64 --version + displayNameName: 'Test signed linux binaries' + workingDirectory: '$(System.DefaultWorkingDirectory)' + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' From 6cb0b95744aaa27cefa488f96d8d3bf8a9559b2d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 08:44:58 +0530 Subject: [PATCH 392/513] test linux binaries --- release-pipelines.yml | 126 +++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index b6bd0c46b..d5edc2375 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -951,75 +951,75 @@ stages: artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Mac Binary' - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # - job: Set_4 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Binaries-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' # SignArtifacts end here From f7d092e4634537f1b773d6e584dd0b7e0281db37 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 08:46:13 +0530 Subject: [PATCH 393/513] test linux binaries --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d5edc2375..361d05430 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1055,7 +1055,7 @@ stages: - script: | azcopy_linux_amd64 --version azcopy_linux_se_amd64 --version - displayNameName: 'Test signed linux binaries' + displayName: 'Test signed linux binaries' workingDirectory: '$(System.DefaultWorkingDirectory)' - template: setup/test_artifacts.yml @@ -1191,7 +1191,7 @@ stages: - script: | azcopy_linux_arm64 --version azcopy_linux_se_arm64 --version - displayNameName: 'Test signed linux binaries' + displayName: 'Test signed linux binaries' workingDirectory: '$(System.DefaultWorkingDirectory)' - template: setup/test_artifacts.yml From 2cffd6e0b70557d42ac1c5ac68ff2d9b0f0c72d9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 09:11:01 +0530 Subject: [PATCH 394/513] test linux binaries --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 361d05430..7535674a6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1050,13 +1050,13 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-binaries-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azcopy-linux*' + item_pattern: 'azCopy-binaries-signed/azcopy-linux*' - script: | azcopy_linux_amd64 --version azcopy_linux_se_amd64 --version displayName: 'Test signed linux binaries' - workingDirectory: '$(System.DefaultWorkingDirectory)' + workingDirectory: '$(System.DefaultWorkingDirectory)/azCopy-binaries-signed' - template: setup/test_artifacts.yml parameters: @@ -1186,13 +1186,13 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-binaries-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azcopy-linux*' + item_pattern: 'azCopy-binaries-signed/azcopy-linux*' - script: | azcopy_linux_arm64 --version azcopy_linux_se_arm64 --version displayName: 'Test signed linux binaries' - workingDirectory: '$(System.DefaultWorkingDirectory)' + workingDirectory: '$(System.DefaultWorkingDirectory)/azCopy-binaries-signed' - template: setup/test_artifacts.yml parameters: From 7830961182c3fda7e7df6b6498760e669781d7e2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 09:25:16 +0530 Subject: [PATCH 395/513] test linux binaries --- release-pipelines.yml | 50 ++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7535674a6..10cc4a8d0 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -114,15 +114,15 @@ stages: cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) displayName: 'List the binaries and add to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux_temp' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" - script: | export GOARCH=amd64 @@ -299,15 +299,15 @@ stages: cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) displayName: 'List the binaries and add to staging directory' - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" - script: | export GOARCH=arm64 @@ -613,9 +613,25 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + mv azCopy-linux-temp/azcopy_linux* . + mv azCopy-linux-temp $(System.DefaultWorkingDirectory) + displayName: 'Add binaries to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push linux binaries to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mv $(System.DefaultWorkingDirectory)/azCopy-linux-temp . sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + md5sum azCopy-linux-temp/* displayName: 'List Artifacts' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | mkdir mariner && chmod 755 mariner From 8215e4b5d2d3dd9406b92b93e812f600657743be Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 09:30:17 +0530 Subject: [PATCH 396/513] test linux binaries --- release-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 10cc4a8d0..afe497eef 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -722,9 +722,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download linux binary for extracting version' inputs: - artifactName: 'azCopy-binaries' + artifactName: 'azCopy-linux-temp' downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + itemPattern: 'azCopy-linux-temp/azcopy_linux_amd64' - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) @@ -772,9 +772,9 @@ stages: workingDirectory: '$(Build.ArtifactStagingDirectory)' - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) if echo "$azcopyOutput" | grep -q "azcopy version"; then version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') From 682b01f1a3a1ea0140cd9b6ff44fea58046955ff Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 10:06:01 +0530 Subject: [PATCH 397/513] test linux binaries --- release-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index afe497eef..385afbc13 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -621,10 +621,11 @@ stages: # Push linux binaries to artifact directory - task: PublishBuildArtifacts@1 inputs: - artifactName: 'azCopy-linux-signed' + artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Artifacts' - script: | + rm -rf azcopy_linux* mv $(System.DefaultWorkingDirectory)/azCopy-linux-temp . sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum azCopy-linux-temp/* From e516edaa7234cfea19e19adcb2523e2dede8bffb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 10:38:37 +0530 Subject: [PATCH 398/513] test linux binaries --- release-pipelines.yml | 123 ++++++++++-------------------------------- 1 file changed, 28 insertions(+), 95 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 385afbc13..0e525cacf 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -114,15 +114,15 @@ stages: cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) displayName: 'List the binaries and add to staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux_temp' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - script: | export GOARCH=amd64 @@ -299,15 +299,15 @@ stages: cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) displayName: 'List the binaries and add to staging directory' - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - script: | export GOARCH=arm64 @@ -609,12 +609,11 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifactName: 'azCopy-linux-temp' + artifactName: 'azCopy-binaries' downloadPath: $(Build.ArtifactStagingDirectory) - script: | - mv azCopy-linux-temp/azcopy_linux* . - mv azCopy-linux-temp $(System.DefaultWorkingDirectory) + mv azCopy-binaries/* . displayName: 'Add binaries to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -623,17 +622,20 @@ stages: inputs: artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Artifacts' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - script: | rm -rf azcopy_linux* - mv $(System.DefaultWorkingDirectory)/azCopy-linux-temp . - sudo ls -lRt $(Build.ArtifactStagingDirectory) + sudo ls -lRt azCopy-linux-temp md5sum azCopy-linux-temp/* displayName: 'List Artifacts' workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | mkdir mariner && chmod 755 mariner cp azCopy-linux-temp/*arm64.rpm mariner @@ -723,13 +725,14 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download linux binary for extracting version' inputs: - artifactName: 'azCopy-linux-temp' + artifactName: 'azCopy-binaries' downloadPath: $(work_dir)/ - itemPattern: 'azCopy-linux-temp/azcopy_linux_amd64' + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md displayName: 'List Artifacts' - script: | @@ -968,76 +971,6 @@ stages: artifactName: 'azCopy-binaries-signed' displayName: 'Publish Signed Mac Binary' - # - job: Set_4 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Binaries-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none - - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # displayName: 'List Artifacts' - - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' - # SignArtifacts end here - stage: TestArtifacts From 0663594436801f2b58b47de1f167f1efeeb08cb9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 11:04:13 +0530 Subject: [PATCH 399/513] test linux binaries --- release-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 0e525cacf..6c42f2d8e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -732,7 +732,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md displayName: 'List Artifacts' - script: | @@ -1000,7 +1000,7 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-binaries-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries-signed/azcopy-linux*' + item_pattern: 'azCopy-binaries-signed/azcopy_linux*' - script: | azcopy_linux_amd64 --version @@ -1136,7 +1136,7 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-binaries-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries-signed/azcopy-linux*' + item_pattern: 'azCopy-binaries-signed/azcopy_linux*' - script: | azcopy_linux_arm64 --version From 372719624bb706463171e1492dccc157d96dd356 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 11:30:12 +0530 Subject: [PATCH 400/513] test linux binaries --- setup/test_artifacts.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 70396f0ed..2cfe5cfe1 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -35,9 +35,5 @@ steps: - script: | ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi displayName: 'List Downloaded Package' workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} From 4a50b48cb44478bad5bd9e42a4ecb66ac5f359e1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 11:55:33 +0530 Subject: [PATCH 401/513] test linux binaries --- release-pipelines.yml | 870 +++++++++++++++++++++--------------------- 1 file changed, 434 insertions(+), 436 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6c42f2d8e..27e88d442 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -411,174 +411,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -692,284 +692,284 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - - script: | - cp azCopy-windows-temp/* . - mv azCopy-windows-temp $(work_dir) - displayName: 'Add signed windows binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' - - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - mv $(work_dir)/azCopy-windows-temp . - rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | - chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # - script: | + # cp azCopy-windows-temp/* . + # mv azCopy-windows-temp $(work_dir) + # displayName: 'Add signed windows binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Artifacts' + + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # mv $(work_dir)/azCopy-windows-temp . + # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + # chmod 755 azCopy-windows-temp/* + # rm -rf azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # - script: | + # chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir -p $windows_amd64 + # mkdir -p $windows_arm64 + # mkdir -p $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p unzipped_amd64 - mkdir -p unzipped_arm64 - unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - echo "AMD64 Folder name: $amd64FolderName" - - arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - # Check if there is exactly one matching directory for amd64 - if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching AMD64 directory: $amd64FolderName" - cd "$amd64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - else - echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - exit 1 - fi - - cd $(Build.ArtifactStagingDirectory) - - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching ARM64 directory: $arm64FolderName" - cd "$arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - else - echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - exit 1 - fi - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p unzipped_amd64 + # mkdir -p unzipped_arm64 + # unzip -o azcopy*amd64*.zip -d unzipped_amd64 + # unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + # echo "AMD64 Folder name: $amd64FolderName" + + # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + # echo "ARM64 Folder name: $arm64FolderName" + + # # Check if there is exactly one matching directory for amd64 + # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching AMD64 directory: $amd64FolderName" + # cd "$amd64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # else + # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + # exit 1 + # fi + + # cd $(Build.ArtifactStagingDirectory) + + # # Check if there is exactly one matching directory for arm64 + # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching ARM64 directory: $arm64FolderName" + # cd "$arm64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # else + # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + # exit 1 + # fi + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries-signed' + # displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -1003,11 +1003,10 @@ stages: item_pattern: 'azCopy-binaries-signed/azcopy_linux*' - script: | - azcopy_linux_amd64 --version - azcopy_linux_se_amd64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version displayName: 'Test signed linux binaries' - workingDirectory: '$(System.DefaultWorkingDirectory)/azCopy-binaries-signed' - + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1139,10 +1138,9 @@ stages: item_pattern: 'azCopy-binaries-signed/azcopy_linux*' - script: | - azcopy_linux_arm64 --version - azcopy_linux_se_arm64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_arm64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_arm64 --version displayName: 'Test signed linux binaries' - workingDirectory: '$(System.DefaultWorkingDirectory)/azCopy-binaries-signed' - template: setup/test_artifacts.yml parameters: From 89375b37ff3ed582dc815256111922e07e6e931a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 12:15:10 +0530 Subject: [PATCH 402/513] test linux binaries --- release-pipelines.yml | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 27e88d442..68e703a12 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1003,6 +1003,7 @@ stages: item_pattern: 'azCopy-binaries-signed/azcopy_linux*' - script: | + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version displayName: 'Test signed linux binaries' @@ -1138,6 +1139,7 @@ stages: item_pattern: 'azCopy-binaries-signed/azcopy_linux*' - script: | + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_arm64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_arm64 --version displayName: 'Test signed linux binaries' @@ -1738,7 +1740,7 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1750,31 +1752,31 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - #azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - #echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - # echo "Azcopy ARM DEB ID: $azcopyArmDeb" + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - # echo "Azcopy ARM RPM $azcopyArmRpmFile" - # echo "Azcopy ARM RPM ID: $azcopyArmRpm" + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" + echo "Azcopy ARM RPM ID: $azcopyArmRpm" # Find the Mariner RPM file and upload marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" is_preview="false" echo "##vso[task.setvariable variable=is_preview]$is_preview" @@ -1794,7 +1796,7 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ @@ -1810,7 +1812,7 @@ stages: fi fi echo "Repo Name: $repoName" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName fi done < <(tail -n +3 ../packages.csv) displayName: 'Publish the repository' From 6f958935417dfd882c6118d1187725e8eee688dd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 12:43:09 +0530 Subject: [PATCH 403/513] test linux binaries --- release-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 68e703a12..cd4189cfc 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1007,10 +1007,10 @@ stages: $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version displayName: 'Test signed linux binaries' - - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' @@ -1144,9 +1144,9 @@ stages: $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_arm64 --version displayName: 'Test signed linux binaries' - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' From eaf647c2916dd1e371437b0da890ee0339cfbdad Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 13:18:40 +0530 Subject: [PATCH 404/513] test linux binaries --- release-pipelines.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index cd4189cfc..1c22e78bd 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1000,9 +1000,10 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-binaries-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries-signed/azcopy_linux*' + item_pattern: 'azCopy-binaries-signed/azcopy_linux*amd64' - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/ chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version @@ -1011,9 +1012,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - script: | sudo dpkg --info azcopy-*x86_64.deb @@ -1136,9 +1137,10 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-binaries-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries-signed/azcopy_linux*' + item_pattern: 'azCopy-binaries-signed/azcopy_linux*arm64' - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries-signed chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_arm64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_arm64 --version @@ -1147,9 +1149,9 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - script: | sudo dpkg --info azcopy-*arm64.deb From 087141b78876c938e0c213bbe79076206d457878 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 13:54:08 +0530 Subject: [PATCH 405/513] test linux binaries --- release-pipelines.yml | 92 +++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1c22e78bd..90a6a5d0a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -977,56 +977,56 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # - job: Set_1 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-binaries-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries-signed/azcopy_linux*amd64' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-binaries-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-binaries-signed/azcopy_linux*amd64' - - script: | - ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/ - chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* - $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version - $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version - displayName: 'Test signed linux binaries' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: '$(System.DefaultWorkingDirectory)' - itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + # - script: | + # ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/ + # chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* + # $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version + # $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version + # displayName: 'Test signed linux binaries' - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-linux-signed' + # downloadPath: '$(System.DefaultWorkingDirectory)' + # itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' + # - script: | + # sudo dpkg --info azcopy-*x86_64.deb + # sudo dpkg -i azcopy-*x86_64.deb + # sudo apt-get install build-essential -y + # displayName: 'Install deb Package' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + # - script: | + # azcopy --version + # azcopy --help + # displayName: 'Check Version and Help' - job: Set_2 timeoutInMinutes: 120 From e0e49740bcd321043c4d16d0a4466f6c1d88d04f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 14:28:37 +0530 Subject: [PATCH 406/513] test linux binaries --- release-pipelines.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 90a6a5d0a..7248ce829 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -292,6 +292,8 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + chmod 755 $(root_dir)/azcopy_linux_se_arm64 + $(root_dir)/azcopy_linux_se_arm64 --version displayName: 'Generate Linux ARM64 SE Integration' - script: | @@ -613,6 +615,10 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + export GOARCH=arm64 + export GOOS=linux + chmod 755 $(Build.ArtifactStagingDirectory)/* + $(Build.ArtifactStagingDirectory)/azcopy_linux_se_arm64 --version mv azCopy-binaries/* . displayName: 'Add binaries to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' From ad9221651ba11e33c24f5c5f0d67dd1758765ce3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 14:56:55 +0530 Subject: [PATCH 407/513] test linux binaries --- release-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7248ce829..99c5745de 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -615,11 +615,13 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + md5sum azCopy-binaries/* + rm -rf azCopy-binaries/*.md + mv azCopy-binaries/* . export GOARCH=arm64 export GOOS=linux chmod 755 $(Build.ArtifactStagingDirectory)/* $(Build.ArtifactStagingDirectory)/azcopy_linux_se_arm64 --version - mv azCopy-binaries/* . displayName: 'Add binaries to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' From 66b390e243e1faf82991da8af914d4110d6f5f87 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 14:57:47 +0530 Subject: [PATCH 408/513] test linux binaries --- release-pipelines.yml | 138 +++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 99c5745de..5b9920a43 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1036,88 +1036,88 @@ stages: # azcopy --help # displayName: 'Check Version and Help' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_tar_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - script: | - sudo apt update - sudo apt --fix-broken install - ldd --version - displayName: "GLIBC Version" + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_tar_amd64: + # agentName: "blobfuse-ubuntu22" + # vmImage: 'Ubuntu-22.04' + # container: 'test-cnt-ubn-22' + # pool: + # vmImage: $(vmImage) + # steps: + # - checkout: none + # - script: | + # sudo apt update + # sudo apt --fix-broken install + # ldd --version + # displayName: "GLIBC Version" - - template: setup/test_artifacts.yml - parameters: - go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + # - template: setup/test_artifacts.yml + # parameters: + # go_version: '$(AZCOPY_GOLANG_VERSION)' + # artifact_name: 'azCopy-linux-signed' + # download_path: '$(System.DefaultWorkingDirectory)' + # item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - - script: | - #!/bin/bash + # - script: | + # #!/bin/bash - # Define the path to the .tar.gz file and the extraction directory - # Define variables - TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # # Define the path to the .tar.gz file and the extraction directory + # # Define variables + # TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) + # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Check if the file exists and is a regular file - if [ ! -f "$TAR_GZ_FILE" ]; then - echo "Error: File $TAR_GZ_FILE does not exist." - exit 1 - fi + # # Check if the file exists and is a regular file + # if [ ! -f "$TAR_GZ_FILE" ]; then + # echo "Error: File $TAR_GZ_FILE does not exist." + # exit 1 + # fi - # Verify the file type - file_type=$(file -b "$TAR_GZ_FILE") - echo "File type: $file_type" + # # Verify the file type + # file_type=$(file -b "$TAR_GZ_FILE") + # echo "File type: $file_type" - if [[ "$file_type" != *"gzip compressed data"* ]]; then - echo "Error: The file is not a valid .tar.gz file" - exit 1 - fi + # if [[ "$file_type" != *"gzip compressed data"* ]]; then + # echo "Error: The file is not a valid .tar.gz file" + # exit 1 + # fi - # Create extraction directory - mkdir -p "$EXTRACT_DIR" + # # Create extraction directory + # mkdir -p "$EXTRACT_DIR" - # Extract the .tar.gz file - tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" - if [ $? -ne 0 ]; then - echo "Error: Failed to extract $TAR_GZ_FILE" - exit 1 - fi + # # Extract the .tar.gz file + # tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + # if [ $? -ne 0 ]; then + # echo "Error: Failed to extract $TAR_GZ_FILE" + # exit 1 + # fi - # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + # # Find the directory matching the pattern + # matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - echo "Found matching directory: ${matching_dirs[0]}" - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi + # # Check if there is exactly one matching directory + # if [ ${#matching_dirs[@]} -eq 1 ]; then + # echo "Found matching directory: ${matching_dirs[0]}" + # cd "${matching_dirs[0]}" + # else + # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + # exit 1 + # fi - # Check if azcopy exists and is executable - if [ ! -x "./azcopy" ]; then - echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" - exit 1 - fi + # # Check if azcopy exists and is executable + # if [ ! -x "./azcopy" ]; then + # echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" + # exit 1 + # fi - # Run azcopy commands - ./azcopy --version - ./azcopy --help + # # Run azcopy commands + # ./azcopy --version + # ./azcopy --help - displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + # displayName: 'Check Version and Help' + # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - job: Set_3 timeoutInMinutes: 120 From b6104335348674d7315c2f4c4e96443bf8daeab3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 15:45:05 +0530 Subject: [PATCH 409/513] test linux binaries --- release-pipelines.yml | 841 ++++++++++++++++++++---------------------- 1 file changed, 409 insertions(+), 432 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 5b9920a43..52672b8e6 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -607,29 +607,6 @@ stages: - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - md5sum azCopy-binaries/* - rm -rf azCopy-binaries/*.md - mv azCopy-binaries/* . - export GOARCH=arm64 - export GOOS=linux - chmod 755 $(Build.ArtifactStagingDirectory)/* - $(Build.ArtifactStagingDirectory)/azcopy_linux_se_arm64 --version - displayName: 'Add binaries to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push linux binaries to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' @@ -700,284 +677,284 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # - script: | - # cp azCopy-windows-temp/* . - # mv azCopy-windows-temp $(work_dir) - # displayName: 'Add signed windows binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Artifacts' - - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # mv $(work_dir)/azCopy-windows-temp . - # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - # chmod 755 azCopy-windows-temp/* - # rm -rf azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # - script: | - # chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir -p $windows_amd64 - # mkdir -p $windows_arm64 - # mkdir -p $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + - script: | + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p unzipped_amd64 - # mkdir -p unzipped_arm64 - # unzip -o azcopy*amd64*.zip -d unzipped_amd64 - # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - # echo "AMD64 Folder name: $amd64FolderName" - - # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - # echo "ARM64 Folder name: $arm64FolderName" - - # # Check if there is exactly one matching directory for amd64 - # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching AMD64 directory: $amd64FolderName" - # cd "$amd64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # else - # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - # exit 1 - # fi - - # cd $(Build.ArtifactStagingDirectory) - - # # Check if there is exactly one matching directory for arm64 - # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching ARM64 directory: $arm64FolderName" - # cd "$arm64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # else - # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - # exit 1 - # fi - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries-signed' - # displayName: 'Publish Signed Mac Binary' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -985,139 +962,139 @@ stages: dependsOn: SignArtifacts condition: succeeded('SignArtifacts') jobs: - # - job: Set_1 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-binaries-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-binaries-signed/azcopy_linux*amd64' + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-binaries' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-binaries/azcopy_linux*amd64' - # - script: | - # ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/ - # chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* - # $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_amd64 --version - # $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_amd64 --version - # displayName: 'Test signed linux binaries' - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-linux-signed' - # downloadPath: '$(System.DefaultWorkingDirectory)' - # itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - # - script: | - # sudo dpkg --info azcopy-*x86_64.deb - # sudo dpkg -i azcopy-*x86_64.deb - # sudo apt-get install build-essential -y - # displayName: 'Install deb Package' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - # - script: | - # azcopy --version - # azcopy --help - # displayName: 'Check Version and Help' + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries/ + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_tar_amd64: - # agentName: "blobfuse-ubuntu22" - # vmImage: 'Ubuntu-22.04' - # container: 'test-cnt-ubn-22' - # pool: - # vmImage: $(vmImage) - # steps: - # - checkout: none - # - script: | - # sudo apt update - # sudo apt --fix-broken install - # ldd --version - # displayName: "GLIBC Version" - - # - template: setup/test_artifacts.yml - # parameters: - # go_version: '$(AZCOPY_GOLANG_VERSION)' - # artifact_name: 'azCopy-linux-signed' - # download_path: '$(System.DefaultWorkingDirectory)' - # item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' - - # - script: | - # #!/bin/bash - - # # Define the path to the .tar.gz file and the extraction directory - # # Define variables - # TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) - # EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # # Check if the file exists and is a regular file - # if [ ! -f "$TAR_GZ_FILE" ]; then - # echo "Error: File $TAR_GZ_FILE does not exist." - # exit 1 - # fi - - # # Verify the file type - # file_type=$(file -b "$TAR_GZ_FILE") - # echo "File type: $file_type" - - # if [[ "$file_type" != *"gzip compressed data"* ]]; then - # echo "Error: The file is not a valid .tar.gz file" - # exit 1 - # fi - - # # Create extraction directory - # mkdir -p "$EXTRACT_DIR" - - # # Extract the .tar.gz file - # tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" - # if [ $? -ne 0 ]; then - # echo "Error: Failed to extract $TAR_GZ_FILE" - # exit 1 - # fi - - # # Find the directory matching the pattern - # matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) - - # # Check if there is exactly one matching directory - # if [ ${#matching_dirs[@]} -eq 1 ]; then - # echo "Found matching directory: ${matching_dirs[0]}" - # cd "${matching_dirs[0]}" - # else - # echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - # exit 1 - # fi - - # # Check if azcopy exists and is executable - # if [ ! -x "./azcopy" ]; then - # echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" - # exit 1 - # fi - - # # Run azcopy commands - # ./azcopy --version - # ./azcopy --help - - # displayName: 'Check Version and Help' - # workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + + - script: | + #!/bin/bash + + # Define the path to the .tar.gz file and the extraction directory + # Define variables + TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Check if the file exists and is a regular file + if [ ! -f "$TAR_GZ_FILE" ]; then + echo "Error: File $TAR_GZ_FILE does not exist." + exit 1 + fi + + # Verify the file type + file_type=$(file -b "$TAR_GZ_FILE") + echo "File type: $file_type" + + if [[ "$file_type" != *"gzip compressed data"* ]]; then + echo "Error: The file is not a valid .tar.gz file" + exit 1 + fi + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Check if azcopy exists and is executable + if [ ! -x "./azcopy" ]; then + echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - job: Set_3 timeoutInMinutes: 120 @@ -1143,15 +1120,15 @@ stages: - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-binaries-signed' + artifact_name: 'azCopy-binaries' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries-signed/azcopy_linux*arm64' + item_pattern: 'azCopy-binaries/azcopy_linux*arm64' - script: | - ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries-signed - chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/* - $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_arm64 --version - $(System.DefaultWorkingDirectory)/azCopy-binaries-signed/azcopy_linux_se_arm64 --version + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_arm64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_arm64 --version displayName: 'Test signed linux binaries' - task: DownloadBuildArtifacts@0 @@ -1531,7 +1508,7 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Binaries Build Artifacts' inputs: - artifactName: 'azCopy-binaries-signed' + artifactName: 'azCopy-binaries' downloadPath: $(Build.ArtifactStagingDirectory) - script: | @@ -1544,12 +1521,12 @@ stages: displayName: 'Create drop.zip file and install azcopy deb package' - script: | - cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed - cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed - echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" + cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries:" ls -l - # Create the zip archive with all files inside azCopy-binaries-signed + # Create the zip archive with all files inside azCopy-binaries zip -r $(Build.ArtifactStagingDirectory)/drop.zip . cd $(Build.ArtifactStagingDirectory) From 19f24db6698de4cd36021a6b4d9a0173c813795b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 15:46:03 +0530 Subject: [PATCH 410/513] test --- release-pipelines.yml | 324 +++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 162 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 52672b8e6..b2e048082 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -413,174 +413,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts From dc2787c171a73ca2c9b10b53dd9f9079ca5eacae Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 4 Jul 2024 16:25:22 +0530 Subject: [PATCH 411/513] test --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index b2e048082..d34d52bd7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1113,6 +1113,7 @@ stages: - checkout: none - script: | sudo apt update + sudo apt-get install libsecret-1-dev sudo apt --fix-broken install -y ldd --version displayName: "GLIBC Version" From 30f25244842c3c6b8ab048636afc8e9a63d08fbd Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:11:33 +0530 Subject: [PATCH 412/513] test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index d34d52bd7..cb949fe26 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1113,7 +1113,7 @@ stages: - checkout: none - script: | sudo apt update - sudo apt-get install libsecret-1-dev + sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install -y ldd --version displayName: "GLIBC Version" From 25217813a488de017f127511adff3c9e30be46ef Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:46:58 +0530 Subject: [PATCH 413/513] test --- release-pipelines.yml | 999 ++++++++++++++++++++++++------------------ 1 file changed, 564 insertions(+), 435 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index cb949fe26..ff17f6ff1 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -413,174 +413,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -677,284 +677,284 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - - script: | - cp azCopy-windows-temp/* . - mv azCopy-windows-temp $(work_dir) - displayName: 'Add signed windows binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Artifacts' - - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - mv $(work_dir)/azCopy-windows-temp . - rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | - chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # - script: | + # cp azCopy-windows-temp/* . + # mv azCopy-windows-temp $(work_dir) + # displayName: 'Add signed windows binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish Signed Artifacts' + + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # mv $(work_dir)/azCopy-windows-temp . + # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + # chmod 755 azCopy-windows-temp/* + # rm -rf azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # - script: | + # chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir -p $windows_amd64 + # mkdir -p $windows_arm64 + # mkdir -p $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p unzipped_amd64 - mkdir -p unzipped_arm64 - unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - echo "AMD64 Folder name: $amd64FolderName" - - arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - # Check if there is exactly one matching directory for amd64 - if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching AMD64 directory: $amd64FolderName" - cd "$amd64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - else - echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - exit 1 - fi - - cd $(Build.ArtifactStagingDirectory) - - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching ARM64 directory: $arm64FolderName" - cd "$arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - else - echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - exit 1 - fi - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Mac Binary' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p unzipped_amd64 + # mkdir -p unzipped_arm64 + # unzip -o azcopy*amd64*.zip -d unzipped_amd64 + # unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + # echo "AMD64 Folder name: $amd64FolderName" + + # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + # echo "ARM64 Folder name: $arm64FolderName" + + # # Check if there is exactly one matching directory for amd64 + # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching AMD64 directory: $amd64FolderName" + # cd "$amd64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # else + # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + # exit 1 + # fi + + # cd $(Build.ArtifactStagingDirectory) + + # # Check if there is exactly one matching directory for arm64 + # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching ARM64 directory: $arm64FolderName" + # cd "$arm64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # else + # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + # exit 1 + # fi + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -1036,7 +1036,7 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*amd64*.tar.gz' - script: | #!/bin/bash @@ -1095,9 +1095,138 @@ stages: displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + - script: | + #!/bin/bash + + # Define the path to the .tar.gz file and the extraction directory + # Define variables + TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + - job: Set_3 timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*arm64*.tar.gz' + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for SE arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_4 + timeoutInMinutes: 120 strategy: matrix: Ubuntu_arm64: @@ -1150,7 +1279,7 @@ stages: azcopy --help displayName: 'Check Version and Help' - - job: Set_4 + - job: Set_5 timeoutInMinutes: 120 strategy: matrix: @@ -1191,7 +1320,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_5 + - job: Set_6 timeoutInMinutes: 120 strategy: matrix: @@ -1237,7 +1366,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_6 + - job: Set_7 timeoutInMinutes: 120 strategy: matrix: @@ -1283,7 +1412,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_7 + - job: Set_8 timeoutInMinutes: 360 strategy: matrix: @@ -1354,7 +1483,7 @@ stages: ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_8 + - job: Set_9 timeoutInMinutes: 120 strategy: matrix: From ef387c54af6dafc917a2256bfcb44a192942c6d3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:50:19 +0530 Subject: [PATCH 414/513] test --- release-pipelines.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ff17f6ff1..c305f5f2b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1092,14 +1092,10 @@ stages: # Run azcopy commands ./azcopy --version ./azcopy --help - displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | - #!/bin/bash - # Define the path to the .tar.gz file and the extraction directory - # Define variables + - script: | TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" @@ -1128,7 +1124,6 @@ stages: # Run azcopy commands ./azcopy --version ./azcopy --help - displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed From 5eb6d1f5281e0da661c0ca4bd2acb2ebab357ab6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:51:48 +0530 Subject: [PATCH 415/513] test --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c305f5f2b..75c7b050b 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1095,7 +1095,7 @@ stages: displayName: 'Check Version and Help' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | + - script: | TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" @@ -1187,7 +1187,7 @@ stages: displayName: 'Check Version and Help for arm64 tar.gz' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - script: | + - script: | TAR_GZ_FILE=$(ls azcopy_linux_se_arm64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" From 912b807daf97ba9e725a146d68c45639f8dd2b50 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:52:48 +0530 Subject: [PATCH 416/513] test --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 75c7b050b..ab03b842c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1129,7 +1129,7 @@ stages: - job: Set_3 timeoutInMinutes: 120 - strategy: + strategy: matrix: Ubuntu_tar_arm64: vmImage: 'Ubuntu-22.04' From c6ef92c81abe05b3a6c2c30a63fe3f5e87ad0c70 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:55:18 +0530 Subject: [PATCH 417/513] test --- release-pipelines.yml | 278 +++++++++++++++++++++--------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index ab03b842c..7719eaa90 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -677,167 +677,167 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' - # - script: | - # cp azCopy-windows-temp/* . - # mv azCopy-windows-temp $(work_dir) - # displayName: 'Add signed windows binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' + - script: | + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # mv $(work_dir)/azCopy-windows-temp . - # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - # chmod 755 azCopy-windows-temp/* - # rm -rf azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' - # - script: | - # chmod 755 $(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-linux-temp/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - # mkdir -p $windows_amd64 - # mkdir -p $windows_arm64 - # mkdir -p $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true - # - script: | - # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' # - job: Set_3 # timeoutInMinutes: 120 From 6d5cfea341b5bcc8274c21e97baef0d81ee94f68 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:57:24 +0530 Subject: [PATCH 418/513] test --- release-pipelines.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 7719eaa90..62fdf6ce7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1039,28 +1039,9 @@ stages: item_pattern: 'azCopy-linux-signed/azcopy_linux_*amd64*.tar.gz' - script: | - #!/bin/bash - - # Define the path to the .tar.gz file and the extraction directory - # Define variables TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Check if the file exists and is a regular file - if [ ! -f "$TAR_GZ_FILE" ]; then - echo "Error: File $TAR_GZ_FILE does not exist." - exit 1 - fi - - # Verify the file type - file_type=$(file -b "$TAR_GZ_FILE") - echo "File type: $file_type" - - if [[ "$file_type" != *"gzip compressed data"* ]]; then - echo "Error: The file is not a valid .tar.gz file" - exit 1 - fi - # Create extraction directory mkdir -p "$EXTRACT_DIR" @@ -1083,12 +1064,6 @@ stages: exit 1 fi - # Check if azcopy exists and is executable - if [ ! -x "./azcopy" ]; then - echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" - exit 1 - fi - # Run azcopy commands ./azcopy --version ./azcopy --help From f28686bb26ada22c3a766590657f7c1c7b71748a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:57:54 +0530 Subject: [PATCH 419/513] test --- release-pipelines.yml | 546 +++++++++++++++++++++--------------------- 1 file changed, 273 insertions(+), 273 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 62fdf6ce7..ba8ba320e 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -413,174 +413,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -839,122 +839,122 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p unzipped_amd64 - # mkdir -p unzipped_arm64 - # unzip -o azcopy*amd64*.zip -d unzipped_amd64 - # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - # echo "AMD64 Folder name: $amd64FolderName" - - # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - # echo "ARM64 Folder name: $arm64FolderName" - - # # Check if there is exactly one matching directory for amd64 - # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching AMD64 directory: $amd64FolderName" - # cd "$amd64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # else - # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - # exit 1 - # fi - - # cd $(Build.ArtifactStagingDirectory) - - # # Check if there is exactly one matching directory for arm64 - # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching ARM64 directory: $arm64FolderName" - # cd "$arm64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # else - # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - # exit 1 - # fi - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Mac Binary' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' # SignArtifacts end here From 3eae591e164e20693dfe44bc1ed47bf47554b6f0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 3 Jul 2024 15:51:05 +0530 Subject: [PATCH 420/513] incorporated review comments --- release-pipeline.yml | 357 +++++++++++++++++++++++---------------- setup/test_artifacts.yml | 4 - 2 files changed, 211 insertions(+), 150 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index f38e6d2b6..86ce0864b 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -292,6 +292,8 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + chmod 755 $(root_dir)/azcopy_linux_se_arm64 + $(root_dir)/azcopy_linux_se_arm64 --version displayName: 'Generate Linux ARM64 SE Integration' - script: | @@ -471,16 +473,6 @@ stages: GOOS: windows CGO_ENABLED: 0 - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe - displayName: 'Remove arm7 binary from staging directory' - - task: PublishBuildArtifacts@1 inputs: artifactName: 'azCopy-windows-temp' @@ -615,7 +607,7 @@ stages: - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' - + - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: @@ -623,9 +615,11 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + rm -rf azcopy_linux* + sudo ls -lRt azCopy-linux-temp + md5sum azCopy-linux-temp/* displayName: 'List Artifacts' + workingDirectory: '$(Build.ArtifactStagingDirectory)' - script: | mkdir mariner && chmod 755 mariner @@ -723,6 +717,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md displayName: 'List Artifacts' - script: | @@ -743,11 +738,27 @@ stages: ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) Pattern: 'azcopy_windows*' + - script: | + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' + # Validate signed images have md5sum changed - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' - script: | chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 @@ -941,86 +952,9 @@ stages: # Push signed images to artifact directory - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Mac Binary' - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Binaries-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' inputs: artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - displayName: 'List Artifacts' - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: 'azcopy_linux*' - DisplayName: 'ESRP CodeSigning azCopy linux binaries' - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)/azCopy-binaries' - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin* - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-binaries/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-binaries/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-binaries/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries-signed' - displayName: 'Publish Signed Artifacts' + displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -1049,9 +983,23 @@ stages: - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' + artifact_name: 'azCopy-binaries' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + item_pattern: 'azCopy-binaries/azcopy_linux*amd64' + + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries/ + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - script: | sudo dpkg --info azcopy-*x86_64.deb @@ -1088,31 +1036,103 @@ stages: go_version: '$(AZCOPY_GOLANG_VERSION)' artifact_name: 'azCopy-linux-signed' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_amd64*.tar.gz' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*amd64*.tar.gz' - script: | - #!/bin/bash - - # Define the path to the .tar.gz file and the extraction directory - # Define variables TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - # Check if the file exists and is a regular file - if [ ! -f "$TAR_GZ_FILE" ]; then - echo "Error: File $TAR_GZ_FILE does not exist." + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" exit 1 fi - # Verify the file type - file_type=$(file -b "$TAR_GZ_FILE") - echo "File type: $file_type" + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_amd64*) - if [[ "$file_type" != *"gzip compressed data"* ]]; then - echo "Error: The file is not a valid .tar.gz file" + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." exit 1 fi + # Run azcopy commands + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*arm64*.tar.gz' + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + # Create extraction directory mkdir -p "$EXTRACT_DIR" @@ -1124,7 +1144,7 @@ stages: fi # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_arm64*) # Check if there is exactly one matching directory if [ ${#matching_dirs[@]} -eq 1 ]; then @@ -1135,9 +1155,36 @@ stages: exit 1 fi - # Check if azcopy exists and is executable - if [ ! -x "./azcopy" ]; then - echo "Error: azcopy not found or not executable in ${matching_dirs[0]}" + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." exit 1 fi @@ -1145,10 +1192,10 @@ stages: ./azcopy --version ./azcopy --help - displayName: 'Check Version and Help' + displayName: 'Check Version and Help for SE arm64 tar.gz' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - job: Set_3 + - job: Set_4 timeoutInMinutes: 120 strategy: matrix: @@ -1165,6 +1212,7 @@ stages: - checkout: none - script: | sudo apt update + sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install -y ldd --version displayName: "GLIBC Version" @@ -1172,9 +1220,23 @@ stages: - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' - artifact_name: 'azCopy-linux-signed' + artifact_name: 'azCopy-binaries' download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + item_pattern: 'azCopy-binaries/azcopy_linux*arm64' + + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_arm64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_arm64 --version + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - script: | sudo dpkg --info azcopy-*arm64.deb @@ -1187,7 +1249,7 @@ stages: azcopy --help displayName: 'Check Version and Help' - - job: Set_4 + - job: Set_5 timeoutInMinutes: 120 strategy: matrix: @@ -1228,7 +1290,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_5 + - job: Set_6 timeoutInMinutes: 120 strategy: matrix: @@ -1274,7 +1336,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_6 + - job: Set_7 timeoutInMinutes: 120 strategy: matrix: @@ -1320,7 +1382,7 @@ stages: azcopy --help displayName: 'Check Version and Help command' - - job: Set_7 + - job: Set_8 timeoutInMinutes: 360 strategy: matrix: @@ -1391,7 +1453,7 @@ stages: ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_8 + - job: Set_9 timeoutInMinutes: 120 strategy: matrix: @@ -1519,6 +1581,9 @@ stages: value: 'azcopyvnextrelease' - name: Test_Mode value: ${{ parameters.test_mode }} + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + steps: @@ -1543,11 +1608,11 @@ stages: - task: DownloadBuildArtifacts@0 displayName: 'Download Binaries Build Artifacts' inputs: - artifactName: 'azCopy-binaries-signed' + artifactName: 'azCopy-binaries' downloadPath: $(Build.ArtifactStagingDirectory) - script: | - cp NOTICE.txt azCopy-binaries-signed/ + sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb @@ -1555,13 +1620,13 @@ stages: sudo apt-get install build-essential -y displayName: 'Create drop.zip file and install azcopy deb package' - #TODO: check once - script: | - cd $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed - echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries-signed:" + cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries:" ls -l - # Create the zip archive with all files inside azCopy-binaries-signed + # Create the zip archive with all files inside azCopy-binaries zip -r $(Build.ArtifactStagingDirectory)/drop.zip . cd $(Build.ArtifactStagingDirectory) @@ -1762,7 +1827,7 @@ stages: - script: | mv azCopy-linux-signed/*tar.gz . - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) @@ -1774,31 +1839,31 @@ stages: workingDirectory: $(Build.ArtifactStagingDirectory)/ - script: | - #azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - #echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - # azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - # echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - # azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - # echo "Azcopy ARM DEB ID: $azcopyArmDeb" + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - # azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - # echo "Azcopy ARM RPM $azcopyArmRpmFile" - # echo "Azcopy ARM RPM ID: $azcopyArmRpm" + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" + echo "Azcopy ARM RPM ID: $azcopyArmRpm" # Find the Mariner RPM file and upload marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - # marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - # echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - # echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - # marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - # echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - # echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" is_preview="false" echo "##vso[task.setvariable variable=is_preview]$is_preview" @@ -1818,14 +1883,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | while IFS=, read -r distro archType repoName releaseName; do - if [[ $ArchType == *"Arm"* ]]; then + if [[ $archType == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1834,7 +1899,7 @@ stages: fi fi echo "Repo Name: $repoName" - #pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName fi done < <(tail -n +3 ../packages.csv) displayName: 'Publish the repository' diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 70396f0ed..2cfe5cfe1 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -35,9 +35,5 @@ steps: - script: | ls -l - result=$(ls -1 | wc -l) - if [ $result -ne 1 ]; then - exit 1 - fi displayName: 'List Downloaded Package' workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} From e95e43d16c6fb3f9441bb4fe6a3a36cb05b85479 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 09:35:23 +0530 Subject: [PATCH 421/513] resolved conflicts --- release-pipeline.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index ac1ff2e66..15d3ddf3d 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1883,23 +1883,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" -<<<<<<< HEAD - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName -======= pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName ->>>>>>> e4ba9bb0e3388bdc1f91423060d83707172d1084 done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | -<<<<<<< HEAD - while IFS=, read -r distro archType repoName releaseName; do - if [[ $archType == *"Arm"* ]]; then -======= while IFS=, read -r distro archetype repoName releaseName; do if [[ $archetype == *"Arm"* ]]; then ->>>>>>> e4ba9bb0e3388bdc1f91423060d83707172d1084 echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then From 17f323d2f7fd54b1512dea43c05ffbf79f88652e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 5 Jul 2024 08:57:54 +0530 Subject: [PATCH 422/513] test --- release-pipeline.yml | 1955 +++++++++++++++++++++++++++++++++++++++++ release-pipelines.yml | 547 ++++++------ 2 files changed, 2229 insertions(+), 273 deletions(-) create mode 100644 release-pipeline.yml diff --git a/release-pipeline.yml b/release-pipeline.yml new file mode 100644 index 000000000..15d3ddf3d --- /dev/null +++ b/release-pipeline.yml @@ -0,0 +1,1955 @@ +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + +parameters: + - name: tag + displayName: 'New Release Tag' + type: string + default: 'azcopy-' + + - name: test_mode + displayName: 'Test Mode' + type: boolean + default: false + + - name: post_release + displayName: 'Post Release on Github' + type: boolean + default: false + + - name: publish_artifacts + displayName: 'Publish Artifacts to Linux Repos' + type: boolean + default: false + + - name: draft + displayName: 'Post as Draft Release' + type: boolean + default: false + + - name: prerelease + displayName: 'Post as PreRelease' + type: boolean + default: false + + - name: update_version + displayName: 'Update Version' + type: boolean + default: false + +# Do not trigger this pipeline automatically +trigger: none +pr: none + +stages: + - stage: BuildArtifacts + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + export GOARCH=amd64 + export GOOS=linux + # Run azcopy --version and capture the output + azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + ls -ltR $(work_dir) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + cp NOTICE.txt pkgDir/usr/bin/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + #TODO: remove this after merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + chmod 777 $(work_dir)/go_installer.sh + sudo ls -lRt $(work_dir)/ + displayName: 'list' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + chmod 755 $(root_dir)/azcopy_linux_se_arm64 + $(root_dir)/azcopy_linux_se_arm64 --version + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - script: | + export GOARCH=arm64 + export GOOS=linux + azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + + # Set the pipeline variable (for Azure DevOps) + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' + + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here + + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + rm -rf azcopy_linux* + sudo ls -lRt azCopy-linux-temp + md5sum azCopy-linux-temp/* + displayName: 'List Artifacts' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' + + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + - script: | + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' + + # SignArtifacts end here + + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-binaries' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-binaries/azcopy_linux*amd64' + + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries/ + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*amd64*.tar.gz' + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + steps: + - checkout: none + - script: | + sudo apt update + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*arm64*.tar.gz' + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for SE arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + - script: | + sudo apt update + sudo apt-get install libsecret-1-dev -y + sudo apt --fix-broken install -y + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-binaries' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-binaries/azcopy_linux*arm64' + + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_arm64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_arm64 --version + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_6 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' + + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + - script: | + sudo yum update -y + sudo yum install git -y + displayName: 'Install Git' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + sudo tdnf update -y + sudo tdnf install git -y + displayName: 'Install dependencies' + + - template: setup/test_artifacts.yml + parameters: + go_version: '$(AZCOPY_GOLANG_VERSION)' + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_8 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + + - job: Set_9 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } + } + displayName: 'Extract Files and Run Version and Help Command on Windows' + # TestArtifacts ends here + + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToContainer + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + - name: Test_Mode + value: ${{ parameters.test_mode }} + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Binaries Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + + sudo apt-get install -y zip + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Create drop.zip file and install azcopy deb package' + + - script: | + cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries:" + ls -l + + # Create the zip archive with all files inside azCopy-binaries + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + cd $(Build.ArtifactStagingDirectory) + + # List the contents of the resulting zip file for verification + unzip -l $(Build.ArtifactStagingDirectory)/drop.zip + + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . + displayName: 'Archive all binaries to drop.zip' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - task: Bash@3 + displayName: 'Install Azure CLI' + inputs: + targetType: 'inline' + script: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=$(azcopy --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + today=$(date +"%Y%m%d") + + if [ $(Test_Mode) = "True" ]; then + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + - job: ReleaseAzcopy + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - ${{ if eq(parameters.post_release, true) }}: + + # add release tags & push to github + #TODO: update github connection and repo name while merging to main + - task: GithubRelease@1 + inputs: + githubConnection: 'dphulkar-msft-github-rel' + repositoryName: 'dphulkar-msft/azure-storage-azcopy' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZCOPY_ESRP_SECRET + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + sudo ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + mv azCopy-linux-signed/*tar.gz . + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + displayName: 'Upload files' + workingDirectory: $(Build.ArtifactStagingDirectory) + + # TODO: replace org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" + echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + + is_preview="false" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + is_preview="true" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + fi + + while IFS=, read -r distro archetype repoName releaseName; do + + # If the package is preview, publish to mariner preview package + if [[ $distro == *"Mariner-2.0"* ]]; then + if [ $is_preview = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + + echo "Uploading packages for $distro" + echo "Repo Name: $repoName" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + while IFS=, read -r distro archetype repoName releaseName; do + if [[ $archetype == *"Arm"* ]]; then + echo "Skipping for ARM type on $distro" + else + if [[ $distro == *"Mariner-2.0"* ]]; then + if [ "$(is_preview)" = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + echo "Repo Name: $repoName" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - stage: UpdateLatestVersion + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - ${{ if eq(parameters.update_version, true) }}: + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' + + # TODO: change org name while merging to main + - script: | + wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) diff --git a/release-pipelines.yml b/release-pipelines.yml index 62fdf6ce7..dd2e7bd12 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -413,174 +413,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -839,122 +839,122 @@ stages: artifactName: 'azCopy-windows-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p unzipped_amd64 - # mkdir -p unzipped_arm64 - # unzip -o azcopy*amd64*.zip -d unzipped_amd64 - # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - # echo "AMD64 Folder name: $amd64FolderName" - - # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - # echo "ARM64 Folder name: $arm64FolderName" - - # # Check if there is exactly one matching directory for amd64 - # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching AMD64 directory: $amd64FolderName" - # cd "$amd64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # else - # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - # exit 1 - # fi - - # cd $(Build.ArtifactStagingDirectory) - - # # Check if there is exactly one matching directory for arm64 - # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching ARM64 directory: $arm64FolderName" - # cd "$arm64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # else - # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - # exit 1 - # fi - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Mac Binary' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -1118,6 +1118,7 @@ stages: - checkout: none - script: | sudo apt update + sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" From 75d83dc22500f226262e50d13db69ca3b923b95c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Sun, 7 Jul 2024 20:06:59 +0530 Subject: [PATCH 423/513] test --- release-pipelines.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index dd2e7bd12..3341e4d61 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -292,8 +292,6 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - chmod 755 $(root_dir)/azcopy_linux_se_arm64 - $(root_dir)/azcopy_linux_se_arm64 --version displayName: 'Generate Linux ARM64 SE Integration' - script: | @@ -615,7 +613,6 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - rm -rf azcopy_linux* sudo ls -lRt azCopy-linux-temp md5sum azCopy-linux-temp/* displayName: 'List Artifacts' @@ -668,7 +665,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' # Push signed images to artifact directory @@ -741,6 +738,7 @@ stages: - script: | cp azCopy-windows-temp/* . mv azCopy-windows-temp $(work_dir) + ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Add signed windows binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -756,7 +754,6 @@ stages: mv $(work_dir)/azCopy-windows-temp . rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -947,6 +944,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) + rm -rf $(Build.ArtifactStagingDirectory)/*.md displayName: 'Copy and list signed mac binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' From 7d608812e07bf03f64f7a30789e031582a35e5a4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 8 Jul 2024 10:50:47 +0530 Subject: [PATCH 424/513] test --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index 3341e4d61..4a16f689a 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -736,6 +736,7 @@ stages: Pattern: 'azcopy_windows*' - script: | + rm -rf azCopy-windows-temp/*.md cp azCopy-windows-temp/* . mv azCopy-windows-temp $(work_dir) ls -ltR $(Build.ArtifactStagingDirectory) From 5ef790b5efcbd1c22ff10257cce062e1a53e17f4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 8 Jul 2024 10:53:43 +0530 Subject: [PATCH 425/513] testes --- release-pipeline.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 15d3ddf3d..4a16f689a 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -292,8 +292,6 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - chmod 755 $(root_dir)/azcopy_linux_se_arm64 - $(root_dir)/azcopy_linux_se_arm64 --version displayName: 'Generate Linux ARM64 SE Integration' - script: | @@ -615,7 +613,6 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - rm -rf azcopy_linux* sudo ls -lRt azCopy-linux-temp md5sum azCopy-linux-temp/* displayName: 'List Artifacts' @@ -668,7 +665,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' # Push signed images to artifact directory @@ -739,8 +736,10 @@ stages: Pattern: 'azcopy_windows*' - script: | + rm -rf azCopy-windows-temp/*.md cp azCopy-windows-temp/* . mv azCopy-windows-temp $(work_dir) + ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Add signed windows binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -756,7 +755,6 @@ stages: mv $(work_dir)/azCopy-windows-temp . rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -947,6 +945,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) + rm -rf $(Build.ArtifactStagingDirectory)/*.md displayName: 'Copy and list signed mac binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -1118,6 +1117,7 @@ stages: - checkout: none - script: | sudo apt update + sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" @@ -1872,7 +1872,7 @@ stages: echo "##vso[task.setvariable variable=is_preview]$is_preview" fi - while IFS=, read -r distro archetype repoName releaseName; do + while IFS=, read -r distro archType repoName releaseName; do # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1883,14 +1883,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro archetype repoName releaseName; do - if [[ $archetype == *"Arm"* ]]; then + while IFS=, read -r distro archType repoName releaseName; do + if [[ $ArchType == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1953,3 +1953,4 @@ stages: python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) + From 754361f1e9328be5c2873e6bff22b90df6702e2f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 8 Jul 2024 11:01:39 +0530 Subject: [PATCH 426/513] tested --- release-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 4a16f689a..ed8acd112 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1872,7 +1872,7 @@ stages: echo "##vso[task.setvariable variable=is_preview]$is_preview" fi - while IFS=, read -r distro archType repoName releaseName; do + while IFS=, read -r distro archeType repoName releaseName; do # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1883,14 +1883,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archeType} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro archType repoName releaseName; do - if [[ $ArchType == *"Arm"* ]]; then + while IFS=, read -r distro archeType repoName releaseName; do + if [[ $archeType == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then From 608e4cb252ec7d951bdb45ea9834a5a2805ca061 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 11:12:55 +0530 Subject: [PATCH 427/513] check updated changes --- release-pipeline.yml | 12 +++++------- setup/releaseVersionUpdate.py | 1 - 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 15d3ddf3d..9dca33b64 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -67,11 +67,11 @@ stages: steps: #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + # - script: | + # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ + # chmod 777 $(work_dir)/go_installer.sh + # sudo ls -lRt $(work_dir)/ + # displayName: 'list' # Custom script to install Go-lang - task: ShellScript@2 @@ -312,8 +312,6 @@ stages: displayName: "Remove binaries from staging directory" - script: | - export GOARCH=arm64 - export GOOS=linux azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) # Extract the version part diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index f994cdfce..f0f88832e 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -3,7 +3,6 @@ import sys import os from xml.dom import minidom -from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient sasUrl = sys.argv[1] From acf9c17f9e065e18ec1bd1810049228aa7e64c45 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 11:52:11 +0530 Subject: [PATCH 428/513] check updated changes --- release-pipeline.yml | 103 +++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 58 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 9dca33b64..5e549e3d1 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -66,13 +66,6 @@ stages: value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - # - script: | - # wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - # chmod 777 $(work_dir)/go_installer.sh - # sudo ls -lRt $(work_dir)/ - # displayName: 'list' - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -123,30 +116,30 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" + + - template: azcopy-version-check-linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_amd64 - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # - script: | + # # Run azcopy --version and capture the output + # azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) + + # # Extract the version part + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - script: | linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" @@ -251,13 +244,6 @@ stages: value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -292,8 +278,6 @@ stages: - script: | GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - chmod 755 $(root_dir)/azcopy_linux_se_arm64 - $(root_dir)/azcopy_linux_se_arm64 --version displayName: 'Generate Linux ARM64 SE Integration' - script: | @@ -310,27 +294,31 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" + + - template: azcopy-version-check-linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_arm64 - - script: | - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) + # - script: | + # azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" + # # Extract the version part + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + # # Set the pipeline variable (for Azure DevOps) + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # else + # echo "Unable to extract version from azcopy output" + # exit 1 + # fi + # else + # echo "azcopy version string not found in output" + # exit 1 + # fi + # displayName: 'Verify and extract the generated build version' - script: | linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" @@ -1610,7 +1598,6 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb From f456ea5628e1a44e537cfb2ed02eaff256a768e2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 12:23:49 +0530 Subject: [PATCH 429/513] check updated changes --- release-pipelines.yml | 87 ++++++++----------------------------------- 1 file changed, 15 insertions(+), 72 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 4a16f689a..8ad05a15f 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -66,13 +66,6 @@ stages: value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -123,30 +116,10 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + + - template: azcopy-version-check-linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_amd64 - script: | linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" @@ -251,13 +224,6 @@ stages: value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -308,29 +274,10 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + + - template: azcopy-version-check-linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_arm64 - script: | linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" @@ -613,6 +560,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + rm -rf azcopy_linux* sudo ls -lRt azCopy-linux-temp md5sum azCopy-linux-temp/* displayName: 'List Artifacts' @@ -665,7 +613,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' # Push signed images to artifact directory @@ -736,10 +684,8 @@ stages: Pattern: 'azcopy_windows*' - script: | - rm -rf azCopy-windows-temp/*.md cp azCopy-windows-temp/* . mv azCopy-windows-temp $(work_dir) - ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Add signed windows binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -755,6 +701,7 @@ stages: mv $(work_dir)/azCopy-windows-temp . rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -945,7 +892,6 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) - rm -rf $(Build.ArtifactStagingDirectory)/*.md displayName: 'Copy and list signed mac binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -1117,7 +1063,6 @@ stages: - checkout: none - script: | sudo apt update - sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" @@ -1612,7 +1557,6 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb @@ -1872,7 +1816,7 @@ stages: echo "##vso[task.setvariable variable=is_preview]$is_preview" fi - while IFS=, read -r distro archType repoName releaseName; do + while IFS=, read -r distro archetype repoName releaseName; do # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1883,14 +1827,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro archType repoName releaseName; do - if [[ $ArchType == *"Arm"* ]]; then + while IFS=, read -r distro archetype repoName releaseName; do + if [[ $archetype == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1953,4 +1897,3 @@ stages: python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) - From 6251abb0ead475079cadaa905d5a42173b4003b4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 12:25:17 +0530 Subject: [PATCH 430/513] check updated changes --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 8ad05a15f..92f57ea27 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -117,7 +117,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - template: azcopy-version-check-linux.yml + - template: setup/azcopy-version-check-linux.yml parameters: azcopy_executable: $(root_dir)/azcopy_linux_amd64 @@ -275,7 +275,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - template: azcopy-version-check-linux.yml + - template: setup/azcopy-version-check-linux.yml parameters: azcopy_executable: $(root_dir)/azcopy_linux_arm64 From 479d8d22ebea32c884c6fc557f23207efa8ebcd1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 12:26:38 +0530 Subject: [PATCH 431/513] check updated changes --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 92f57ea27..caed6f1e8 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -117,7 +117,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - template: setup/azcopy-version-check-linux.yml + - template: setup/azcopy_version_check_linux.yml parameters: azcopy_executable: $(root_dir)/azcopy_linux_amd64 @@ -275,7 +275,7 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - template: setup/azcopy-version-check-linux.yml + - template: setup/azcopy_version_check_linux.yml parameters: azcopy_executable: $(root_dir)/azcopy_linux_arm64 From aa3c83461bc8158598fe3a93deb619ff3bb084ec Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 12:32:49 +0530 Subject: [PATCH 432/513] check updated changes --- setup/azcopy_version_check_linux.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 setup/azcopy_version_check_linux.yml diff --git a/setup/azcopy_version_check_linux.yml b/setup/azcopy_version_check_linux.yml new file mode 100644 index 000000000..66849f4f3 --- /dev/null +++ b/setup/azcopy_version_check_linux.yml @@ -0,0 +1,23 @@ +parameters: + azcopy_executable: '' + +steps: + - script: | + # Run azcopy --version and capture the output + azcopyOutput=$(${{ parameters.azcopy_executable }} --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' \ No newline at end of file From b396292c885e0507d0c34b25508e8971dec1339d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 12:38:01 +0530 Subject: [PATCH 433/513] check updated changes --- release-pipelines.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index caed6f1e8..c13b99f69 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -66,6 +66,17 @@ stages: value: $(root_dir)/archives steps: + #TODO: Change the org name after merging to main + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -224,6 +235,17 @@ stages: value: $(root_dir)/archives steps: + #TODO: Change the org name after merging to main + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + # Custom script to install Go-lang - task: ShellScript@2 inputs: From f8adf23c2d6da292c3b920d2d26aa87d17263168 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 12:43:53 +0530 Subject: [PATCH 434/513] test --- release-pipelines.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index c13b99f69..3013f1048 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -66,16 +66,16 @@ stages: value: $(root_dir)/archives steps: - #TODO: Change the org name after merging to main - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(System.DefaultWorkingDirectory) + # #TODO: Change the org name after merging to main + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(System.DefaultWorkingDirectory) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy # Custom script to install Go-lang - task: ShellScript@2 @@ -235,16 +235,16 @@ stages: value: $(root_dir)/archives steps: - #TODO: Change the org name after merging to main - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + # #TODO: Change the org name after merging to main + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy # Custom script to install Go-lang - task: ShellScript@2 From ec7d54e454704950edc2c5350935c78af62fc981 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 14:43:54 +0530 Subject: [PATCH 435/513] test --- release-pipelines.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 3013f1048..1c8909d0c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -66,11 +66,8 @@ stages: value: $(root_dir)/archives steps: - # #TODO: Change the org name after merging to main - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(System.DefaultWorkingDirectory) + - checkout: self + displayName: 'Checkout repository' # - script: | # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` @@ -235,11 +232,8 @@ stages: value: $(root_dir)/archives steps: - # #TODO: Change the org name after merging to main - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(System.DefaultWorkingDirectory) + - checkout: self + displayName: 'Checkout repository' # - script: | # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` From ac29ade9a437bef82e676a58f3738ba51031378e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 14:53:31 +0530 Subject: [PATCH 436/513] test --- release-pipelines.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1c8909d0c..6e03720af 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -66,13 +66,22 @@ stages: value: $(root_dir)/archives steps: - - checkout: self - displayName: 'Checkout repository' + # - checkout: self + # displayName: 'Checkout repository' + + - checkout: none - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + # list commits from past 12hrs + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + git --no-pager log --since="12 hours ago" --stat + displayName: 'List Commits' + workingDirectory: $(work_dir) # Custom script to install Go-lang - task: ShellScript@2 From 354b4a67dca02ecae1434052c4686694a7c67649 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 14:58:54 +0530 Subject: [PATCH 437/513] test checkout self --- release-pipelines.yml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6e03720af..630c754d1 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -66,22 +66,27 @@ stages: value: $(root_dir)/archives steps: - # - checkout: self - # displayName: 'Checkout repository' - - - checkout: none + - checkout: self + displayName: 'Checkout repository' - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) + ls -al $(Build.SourcesDirectory) # List files to confirm the script is there + chmod +x $(Build.SourcesDirectory)/go_installer.sh + displayName: 'Run go_installer.sh script' + + # - checkout: none + + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(root_dir) - # list commits from past 12hrs - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - git --no-pager log --since="12 hours ago" --stat - displayName: 'List Commits' - workingDirectory: $(work_dir) + # # list commits from past 12hrs + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # git --no-pager log --since="12 hours ago" --stat + # displayName: 'List Commits' + # workingDirectory: $(work_dir) # Custom script to install Go-lang - task: ShellScript@2 From d39dccfe1c2565c65b13aa626cbbfdbef058b156 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 15:03:22 +0530 Subject: [PATCH 438/513] test checkout self --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 630c754d1..153088856 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -61,7 +61,7 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + value: '$(Build.SourcesDirectory)/azure-storage-azcopy' - name: archives value: $(root_dir)/archives @@ -241,7 +241,7 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + value: '$(Build.SourcesDirectory)/azure-storage-azcopy' - name: archives value: $(root_dir)/archives From 13b5668f9667938725d4cbf3fcdc384756bf424c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 15:10:06 +0530 Subject: [PATCH 439/513] test checkout self --- release-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 153088856..1fc3f5e48 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -61,7 +61,7 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(Build.SourcesDirectory)/azure-storage-azcopy' + value: '$(Build.SourcesDirectory)' - name: archives value: $(root_dir)/archives @@ -241,7 +241,7 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(Build.SourcesDirectory)/azure-storage-azcopy' + value: '$(Build.SourcesDirectory)' - name: archives value: $(root_dir)/archives From 4db7d6a481663aeb6f69d6492417aae5f5f17fe6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 15:23:13 +0530 Subject: [PATCH 440/513] test checkout self --- release-pipelines.yml | 24 ------------------------ setup/test_artifacts.yml | 20 +++++++++++--------- 2 files changed, 11 insertions(+), 33 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 1fc3f5e48..6500298d9 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -69,25 +69,6 @@ stages: - checkout: self displayName: 'Checkout repository' - - script: | - ls -al $(Build.SourcesDirectory) # List files to confirm the script is there - chmod +x $(Build.SourcesDirectory)/go_installer.sh - displayName: 'Run go_installer.sh script' - - # - checkout: none - - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(root_dir) - - # # list commits from past 12hrs - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # git --no-pager log --since="12 hours ago" --stat - # displayName: 'List Commits' - # workingDirectory: $(work_dir) - # Custom script to install Go-lang - task: ShellScript@2 inputs: @@ -249,11 +230,6 @@ stages: - checkout: self displayName: 'Checkout repository' - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy - # Custom script to install Go-lang - task: ShellScript@2 inputs: diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 2cfe5cfe1..626156425 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -9,20 +9,22 @@ parameters: type: string steps: + - checkout: self + displayName: 'Checkout repository' #TODO: Change the org name after merging to main - - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(System.DefaultWorkingDirectory) + # - script: | + # git clone https://github.com/dphulkar-msft/azure-storage-azcopy + # displayName: 'Checkout Code' + # workingDirectory: $(System.DefaultWorkingDirectory) - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + # - script: | + # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + # displayName: 'Checkout Branch' + # workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy - task: ShellScript@2 inputs: - scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" + scriptPath: "$(Build.SourcesDirectory)/go_installer.sh" args: "$(System.DefaultWorkingDirectory)/ ${{ parameters.go_version }}" displayName: "GoTool Custom Setup" From 3f3a2432323492e7c2dd36e75e9ecd848fff208a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 15:26:27 +0530 Subject: [PATCH 441/513] test artifact checkout self --- release-pipelines.yml | 14 +++++++------- setup/test_artifacts.yml | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6500298d9..fc4280feb 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -924,7 +924,7 @@ stages: pool: vmImage: $(vmImage) steps: - - checkout: none + - checkout: self - script: | sudo apt update sudo apt --fix-broken install @@ -975,7 +975,7 @@ stages: pool: vmImage: $(vmImage) steps: - - checkout: none + - checkout: self - script: | sudo apt update sudo apt --fix-broken install @@ -1066,7 +1066,7 @@ stages: demands: - ImageOverride -equals $(AgentName) steps: - - checkout: none + - checkout: self - script: | sudo apt update sudo apt --fix-broken install @@ -1160,7 +1160,7 @@ stages: - ImageOverride -equals $(AgentName) steps: - - checkout: none + - checkout: self - script: | sudo apt update sudo apt-get install libsecret-1-dev -y @@ -1215,7 +1215,7 @@ stages: - ImageOverride -equals $(agentName) steps: - - checkout: none + - checkout: self - script: | sudo yum update -y sudo yum install git -y @@ -1256,7 +1256,7 @@ stages: - ImageOverride -equals $(agentName) steps: - - checkout: none + - checkout: self - script: | sudo yum update -y sudo yum install git -y @@ -1303,7 +1303,7 @@ stages: - ImageOverride -equals $(AgentName) steps: - - checkout: none + - checkout: self - script: | sudo tdnf update -y diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 626156425..b8d8d1b8d 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -9,8 +9,8 @@ parameters: type: string steps: - - checkout: self - displayName: 'Checkout repository' + # - checkout: self + # displayName: 'Checkout repository' #TODO: Change the org name after merging to main # - script: | # git clone https://github.com/dphulkar-msft/azure-storage-azcopy From 8afad0b6b358c66a63acb34b6adfec0504a46ce3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 15:44:36 +0530 Subject: [PATCH 442/513] test all --- release-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipelines.yml b/release-pipelines.yml index fc4280feb..6bcd4c260 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1069,6 +1069,7 @@ stages: - checkout: self - script: | sudo apt update + sudo apt-get install libsecret-1-dev sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" From ecf7b19cfe757e97529bd160f02bfca62ce3a289 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 16:22:28 +0530 Subject: [PATCH 443/513] test all --- release-pipelines.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 6bcd4c260..87566e09c 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1216,12 +1216,13 @@ stages: - ImageOverride -equals $(agentName) steps: - - checkout: self - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' + - checkout: self + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1257,12 +1258,13 @@ stages: - ImageOverride -equals $(agentName) steps: - - checkout: self - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' + - checkout: self + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1304,12 +1306,12 @@ stages: - ImageOverride -equals $(AgentName) steps: - - checkout: self - - script: | sudo tdnf update -y sudo tdnf install git -y displayName: 'Install dependencies' + + - checkout: self - template: setup/test_artifacts.yml parameters: From c542ab5ffe33d10db053d388804ff8dce78b2152 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 18:28:55 +0530 Subject: [PATCH 444/513] test all --- release-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 87566e09c..2e823da47 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -1069,7 +1069,7 @@ stages: - checkout: self - script: | sudo apt update - sudo apt-get install libsecret-1-dev + sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" From 01bf09cfca504e60bd2eb87edd4c5c115daea53c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 20:12:41 +0530 Subject: [PATCH 445/513] test all --- release-pipelines.yml | 28 ++++++++++++++++++---------- setup/test_artifacts.yml | 21 ++++++++++----------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/release-pipelines.yml b/release-pipelines.yml index 2e823da47..05b57c7d7 100644 --- a/release-pipelines.yml +++ b/release-pipelines.yml @@ -924,7 +924,8 @@ stages: pool: vmImage: $(vmImage) steps: - - checkout: self + - checkout: none + - script: | sudo apt update sudo apt --fix-broken install @@ -975,7 +976,7 @@ stages: pool: vmImage: $(vmImage) steps: - - checkout: self + - checkout: none - script: | sudo apt update sudo apt --fix-broken install @@ -1066,14 +1067,15 @@ stages: demands: - ImageOverride -equals $(AgentName) steps: - - checkout: self + - checkout: none + - script: | sudo apt update sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" - + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1161,7 +1163,8 @@ stages: - ImageOverride -equals $(AgentName) steps: - - checkout: self + - checkout: none + - script: | sudo apt update sudo apt-get install libsecret-1-dev -y @@ -1216,12 +1219,17 @@ stages: - ImageOverride -equals $(agentName) steps: + - checkout: none + - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' - - checkout: self + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" - template: setup/test_artifacts.yml parameters: @@ -1258,13 +1266,13 @@ stages: - ImageOverride -equals $(agentName) steps: + - checkout: none + - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' - - checkout: self - - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1306,12 +1314,12 @@ stages: - ImageOverride -equals $(AgentName) steps: + - checkout: none + - script: | sudo tdnf update -y sudo tdnf install git -y displayName: 'Install dependencies' - - - checkout: self - template: setup/test_artifacts.yml parameters: diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index b8d8d1b8d..08c449b1e 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -9,22 +9,21 @@ parameters: type: string steps: - # - checkout: self - # displayName: 'Checkout repository' + #TODO: Change the org name after merging to main - # - script: | - # git clone https://github.com/dphulkar-msft/azure-storage-azcopy - # displayName: 'Checkout Code' - # workingDirectory: $(System.DefaultWorkingDirectory) + - script: | + git clone https://github.com/dphulkar-msft/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(System.DefaultWorkingDirectory) - # - script: | - # git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - # displayName: 'Checkout Branch' - # workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(System.DefaultWorkingDirectory)/azure-storage-azcopy - task: ShellScript@2 inputs: - scriptPath: "$(Build.SourcesDirectory)/go_installer.sh" + scriptPath: "$(System.DefaultWorkingDirectory)/go_installer.sh" args: "$(System.DefaultWorkingDirectory)/ ${{ parameters.go_version }}" displayName: "GoTool Custom Setup" From b5f191c5c1edb86f5d8f64980484ef047caf1245 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 20:18:45 +0530 Subject: [PATCH 446/513] incorporated review comments --- release-pipeline.yml | 106 +++++++++------------------ setup/azcopy_version_check_linux.yml | 23 ++++++ setup/releaseVersionUpdate.py | 1 - 3 files changed, 56 insertions(+), 74 deletions(-) create mode 100644 setup/azcopy_version_check_linux.yml diff --git a/release-pipeline.yml b/release-pipeline.yml index ed8acd112..05b57c7d7 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -61,17 +61,13 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + value: '$(Build.SourcesDirectory)' - name: archives value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + - checkout: self + displayName: 'Checkout repository' # Custom script to install Go-lang - task: ShellScript@2 @@ -123,30 +119,10 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - - script: | - export GOARCH=amd64 - export GOOS=linux - # Run azcopy --version and capture the output - azcopyOutput=$($(root_dir)/azcopy_linux_amd64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + + - template: setup/azcopy_version_check_linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_amd64 - script: | linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" @@ -246,17 +222,13 @@ stages: - name: root_dir value: '$(System.DefaultWorkingDirectory)' - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + value: '$(Build.SourcesDirectory)' - name: archives value: $(root_dir)/archives steps: - #TODO: remove this after merging to main - - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/az-pipelineTest/go_installer.sh -P $(work_dir)/ - chmod 777 $(work_dir)/go_installer.sh - sudo ls -lRt $(work_dir)/ - displayName: 'list' + - checkout: self + displayName: 'Checkout repository' # Custom script to install Go-lang - task: ShellScript@2 @@ -308,29 +280,10 @@ stages: - script: | rm -rf $(Build.ArtifactStagingDirectory)/azcopy* displayName: "Remove binaries from staging directory" - - - script: | - export GOARCH=arm64 - export GOOS=linux - azcopyOutput=$($(root_dir)/azcopy_linux_arm64 --version) - - # Extract the version part - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - - # Set the pipeline variable (for Azure DevOps) - echo "##vso[task.setvariable variable=azcopy_version]$version" - else - echo "Unable to extract version from azcopy output" - exit 1 - fi - else - echo "azcopy version string not found in output" - exit 1 - fi - displayName: 'Verify and extract the generated build version' + + - template: setup/azcopy_version_check_linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_arm64 - script: | linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" @@ -613,6 +566,7 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | + rm -rf azcopy_linux* sudo ls -lRt azCopy-linux-temp md5sum azCopy-linux-temp/* displayName: 'List Artifacts' @@ -665,7 +619,7 @@ stages: - script: | sudo ls -lRt $(Build.ArtifactStagingDirectory) md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* displayName: 'List Signed Artifacts' # Push signed images to artifact directory @@ -736,10 +690,8 @@ stages: Pattern: 'azcopy_windows*' - script: | - rm -rf azCopy-windows-temp/*.md cp azCopy-windows-temp/* . mv azCopy-windows-temp $(work_dir) - ls -ltR $(Build.ArtifactStagingDirectory) displayName: 'Add signed windows binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -755,6 +707,7 @@ stages: mv $(work_dir)/azCopy-windows-temp . rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md displayName: 'Make Artifacts executable' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -945,7 +898,6 @@ stages: rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) - rm -rf $(Build.ArtifactStagingDirectory)/*.md displayName: 'Copy and list signed mac binary to staging directory' workingDirectory: '$(Build.ArtifactStagingDirectory)' @@ -973,6 +925,7 @@ stages: vmImage: $(vmImage) steps: - checkout: none + - script: | sudo apt update sudo apt --fix-broken install @@ -1115,13 +1068,14 @@ stages: - ImageOverride -equals $(AgentName) steps: - checkout: none + - script: | sudo apt update sudo apt-get install libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" - + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1210,6 +1164,7 @@ stages: steps: - checkout: none + - script: | sudo apt update sudo apt-get install libsecret-1-dev -y @@ -1265,11 +1220,17 @@ stages: steps: - checkout: none + - script: | sudo yum update -y sudo yum install git -y displayName: 'Install Git' + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + - template: setup/test_artifacts.yml parameters: go_version: '$(AZCOPY_GOLANG_VERSION)' @@ -1306,6 +1267,7 @@ stages: steps: - checkout: none + - script: | sudo yum update -y sudo yum install git -y @@ -1353,7 +1315,7 @@ stages: steps: - checkout: none - + - script: | sudo tdnf update -y sudo tdnf install git -y @@ -1612,7 +1574,6 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - script: | - sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . sudo dpkg --info azcopy-*x86_64.deb @@ -1872,7 +1833,7 @@ stages: echo "##vso[task.setvariable variable=is_preview]$is_preview" fi - while IFS=, read -r distro archeType repoName releaseName; do + while IFS=, read -r distro archetype repoName releaseName; do # If the package is preview, publish to mariner preview package if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1883,14 +1844,14 @@ stages: echo "Uploading packages for $distro" echo "Repo Name: $repoName" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archeType} $repoName $releaseName + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName done < <(tail -n +3 ../packages.csv) displayName: 'Add uploaded packages to repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - script: | - while IFS=, read -r distro archeType repoName releaseName; do - if [[ $archeType == *"Arm"* ]]; then + while IFS=, read -r distro archetype repoName releaseName; do + if [[ $archetype == *"Arm"* ]]; then echo "Skipping for ARM type on $distro" else if [[ $distro == *"Mariner-2.0"* ]]; then @@ -1953,4 +1914,3 @@ stages: python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" displayName: 'Updating version number' workingDirectory: $(root_dir) - diff --git a/setup/azcopy_version_check_linux.yml b/setup/azcopy_version_check_linux.yml new file mode 100644 index 000000000..66849f4f3 --- /dev/null +++ b/setup/azcopy_version_check_linux.yml @@ -0,0 +1,23 @@ +parameters: + azcopy_executable: '' + +steps: + - script: | + # Run azcopy --version and capture the output + azcopyOutput=$(${{ parameters.azcopy_executable }} --version) + + # Extract the version part + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + else + echo "Unable to extract version from azcopy output" + exit 1 + fi + else + echo "azcopy version string not found in output" + exit 1 + fi + displayName: 'Verify and extract the generated build version' \ No newline at end of file diff --git a/setup/releaseVersionUpdate.py b/setup/releaseVersionUpdate.py index f994cdfce..f0f88832e 100644 --- a/setup/releaseVersionUpdate.py +++ b/setup/releaseVersionUpdate.py @@ -3,7 +3,6 @@ import sys import os from xml.dom import minidom -from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient sasUrl = sys.argv[1] From 87e7540084dab76daae8f08f2e46200df8e3bd35 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 9 Jul 2024 20:19:48 +0530 Subject: [PATCH 447/513] test all --- setup/test_artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 08c449b1e..d215bab4b 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -23,7 +23,7 @@ steps: - task: ShellScript@2 inputs: - scriptPath: "$(System.DefaultWorkingDirectory)/go_installer.sh" + scriptPath: "$(System.DefaultWorkingDirectory)/azure-storage-azcopy/go_installer.sh" args: "$(System.DefaultWorkingDirectory)/ ${{ parameters.go_version }}" displayName: "GoTool Custom Setup" From cc33038c53013698801a27c4f21bf3a9adb807db Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 10 Jul 2024 13:22:30 +0530 Subject: [PATCH 448/513] incorporated review comments --- release-pipeline.yml | 10 ++++------ setup/test_artifacts.yml | 3 +-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 05b57c7d7..2d87f23d9 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -150,7 +150,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + fpm -s dir -t deb -n azcopy -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -310,7 +310,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + fpm -s dir -t deb -n azcopy -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -1792,9 +1792,8 @@ stages: displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) - # TODO: replace org name while merging to main - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv cat ./packages.csv displayName: 'Fetch packages.csv' workingDirectory: $(Build.ArtifactStagingDirectory)/ @@ -1902,9 +1901,8 @@ stages: azcopy --version displayName: 'Installing azcopy' - # TODO: change org name while merging to main - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py ls -l displayName: 'Getting Python script' workingDirectory: $(root_dir) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 2cfe5cfe1..790a729b5 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -9,9 +9,8 @@ parameters: type: string steps: - #TODO: Change the org name after merging to main - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy + git clone https://github.com/Azure/azure-storage-azcopy displayName: 'Checkout Code' workingDirectory: $(System.DefaultWorkingDirectory) From 27adcaf83586090f0b8a068a54ed86135bcb26d7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 10 Jul 2024 13:22:30 +0530 Subject: [PATCH 449/513] incorporated review comments --- release-pipeline.yml | 16 ++++++---------- setup/test_artifacts.yml | 3 +-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 05b57c7d7..f924c8bea 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -150,7 +150,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + fpm -s dir -t deb -n azcopy -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -310,7 +310,7 @@ stages: # using fpm tool for packaging of our binary & performing post-install operations # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - script: | - fpm -s dir -t deb -n azcopy --rpm-digest sha256 -C pkgDir/ \ + fpm -s dir -t deb -n azcopy -C pkgDir/ \ -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ --maintainer "Azcopy v-Team " \ --url "https://github.com/Azure/azure-storage-azcopy" \ @@ -1691,12 +1691,10 @@ stages: - ${{ if eq(parameters.post_release, true) }}: - # add release tags & push to github - #TODO: update github connection and repo name while merging to main - task: GithubRelease@1 inputs: - githubConnection: 'dphulkar-msft-github-rel' - repositoryName: 'dphulkar-msft/azure-storage-azcopy' + githubConnection: 'azcopy-github-connection' + repositoryName: 'Azure/azure-storage-azcopy' action: 'edit' target: '$(Build.SourceVersion)' @@ -1792,9 +1790,8 @@ stages: displayName: 'Upload files' workingDirectory: $(Build.ArtifactStagingDirectory) - # TODO: replace org name while merging to main - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv cat ./packages.csv displayName: 'Fetch packages.csv' workingDirectory: $(Build.ArtifactStagingDirectory)/ @@ -1902,9 +1899,8 @@ stages: azcopy --version displayName: 'Installing azcopy' - # TODO: change org name while merging to main - script: | - wget https://raw.githubusercontent.com/dphulkar-msft/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py ls -l displayName: 'Getting Python script' workingDirectory: $(root_dir) diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 2cfe5cfe1..790a729b5 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -9,9 +9,8 @@ parameters: type: string steps: - #TODO: Change the org name after merging to main - script: | - git clone https://github.com/dphulkar-msft/azure-storage-azcopy + git clone https://github.com/Azure/azure-storage-azcopy displayName: 'Checkout Code' workingDirectory: $(System.DefaultWorkingDirectory) From 86b4c3a9569ca6609ea5afdecac396b1dd667b1a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 12 Jul 2024 15:59:29 +0530 Subject: [PATCH 450/513] Added flag for releasing to public container --- release-pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index ed301ec0a..7ec27fb3e 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -7,8 +7,8 @@ parameters: type: string default: 'azcopy-' - - name: test_mode - displayName: 'Test Mode' + - name: publish_to_container + displayName: 'Publish To Container' type: boolean default: false @@ -1542,8 +1542,8 @@ stages: value: 'azcopyprivatedrops' - name: Release_Account_Name value: 'azcopyvnextrelease' - - name: Test_Mode - value: ${{ parameters.test_mode }} + - name: Publish_To_Container + value: ${{ parameters.publish_to_container }} - name: work_dir value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' @@ -1619,10 +1619,10 @@ stages: today=$(date +"%Y%m%d") - if [ $(Test_Mode) = "True" ]; then - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - else + if [ $(Publish_To_Container) = "True" ]; then container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" fi echo "Artifacts will be uploaded to: $container_url" From 62581f340951f295dee555a873707f05a3e8fcc0 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 19 Jul 2024 10:59:29 +0530 Subject: [PATCH 451/513] changes test name --- e2etest/zt_amanaged_disks_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index 22e45f3b1..9c3c5b88d 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -22,10 +22,11 @@ package e2etest import ( "flag" - "github.com/Azure/azure-storage-azcopy/v10/common" "runtime" "testing" "time" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) // Purpose: Tests for the special cases that relate to moving managed disks (default local VHD to page blob; special handling for @@ -118,7 +119,7 @@ func TestManagedDisks_SnapshotOAuth(t *testing.T) { } // Service issue causes occasional flickers in feature functionality; enough that testing is problematic. Temporarily disabled until issue is resolved. -func TestManagedDisks_OAuthRequired(t *testing.T) { +func TestAAA(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("Limit runs to Linux so no simultaneous runs occur") return From 3b0c7556ca5ca8d044ee3c947b0c420dc3261f04 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 19 Jul 2024 13:26:16 +0530 Subject: [PATCH 452/513] changes test name --- e2etest/zt_amanaged_disks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index 9c3c5b88d..fcf68a8a1 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -138,7 +138,7 @@ func TestAAA(t *testing.T) { &hooks{ beforeRunJob: func(h hookHelper) { // try giving the service some time to think - time.Sleep(time.Second * 30) + time.Sleep(time.Second * 5) }, }, testFiles{ From 345db352d745c746efc4c413dbc753d4836a7554 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 19 Jul 2024 13:27:55 +0530 Subject: [PATCH 453/513] changes test name --- e2etest/zt_amanaged_disks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index fcf68a8a1..ac45d9ca7 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -119,7 +119,7 @@ func TestManagedDisks_SnapshotOAuth(t *testing.T) { } // Service issue causes occasional flickers in feature functionality; enough that testing is problematic. Temporarily disabled until issue is resolved. -func TestAAA(t *testing.T) { +func TestAaa(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("Limit runs to Linux so no simultaneous runs occur") return From d41c7eb089854175ff96d4f95080d308f5dd1904 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 19 Jul 2024 16:02:08 +0530 Subject: [PATCH 454/513] changes test name --- e2etest/zt_amanaged_disks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index ac45d9ca7..578f0bbdb 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -119,7 +119,7 @@ func TestManagedDisks_SnapshotOAuth(t *testing.T) { } // Service issue causes occasional flickers in feature functionality; enough that testing is problematic. Temporarily disabled until issue is resolved. -func TestAaa(t *testing.T) { +func TestManagedDisks_Aaa(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("Limit runs to Linux so no simultaneous runs occur") return From fc2b59b7429ed08c74ca173fdb9d3d11bd4b5ab8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 19 Jul 2024 17:00:11 +0530 Subject: [PATCH 455/513] remove code signature file as it is not required --- release-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipeline.yml b/release-pipeline.yml index 7ec27fb3e..ec5be4bae 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -690,6 +690,7 @@ stages: Pattern: 'azcopy_windows*' - script: | + rm -rf azCopy-windows-temp/*.md cp azCopy-windows-temp/* . mv azCopy-windows-temp $(work_dir) displayName: 'Add signed windows binary to staging directory' From 44c324451a75a906d036e26a522cd5dfa22288f8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 22 Jul 2024 13:10:34 +0530 Subject: [PATCH 456/513] rolled back test name --- e2etest/zt_amanaged_disks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index 578f0bbdb..fdbc3e732 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -119,7 +119,7 @@ func TestManagedDisks_SnapshotOAuth(t *testing.T) { } // Service issue causes occasional flickers in feature functionality; enough that testing is problematic. Temporarily disabled until issue is resolved. -func TestManagedDisks_Aaa(t *testing.T) { +func TestManagedDisks_OAuthRequired(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("Limit runs to Linux so no simultaneous runs occur") return From f598685fd0ef604b32a16e07561e509141020d1e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 22 Jul 2024 13:10:34 +0530 Subject: [PATCH 457/513] rolled back test name --- e2etest/zt_amanaged_disks_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index 578f0bbdb..f708abc0e 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -119,7 +119,7 @@ func TestManagedDisks_SnapshotOAuth(t *testing.T) { } // Service issue causes occasional flickers in feature functionality; enough that testing is problematic. Temporarily disabled until issue is resolved. -func TestManagedDisks_Aaa(t *testing.T) { +func TestManagedDisks_OAuthRequired(t *testing.T) { if runtime.GOOS != "linux" { t.Skip("Limit runs to Linux so no simultaneous runs occur") return @@ -138,7 +138,7 @@ func TestManagedDisks_Aaa(t *testing.T) { &hooks{ beforeRunJob: func(h hookHelper) { // try giving the service some time to think - time.Sleep(time.Second * 5) + time.Sleep(time.Second * 30) }, }, testFiles{ From 2c467174de5b32aefbf3c831ff73b341084b7638 Mon Sep 17 00:00:00 2001 From: dphulkar-msft <166800991+dphulkar-msft@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:01:11 +0530 Subject: [PATCH 458/513] Update zt_amanaged_disks_test.go --- e2etest/zt_amanaged_disks_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_amanaged_disks_test.go index f708abc0e..22e45f3b1 100644 --- a/e2etest/zt_amanaged_disks_test.go +++ b/e2etest/zt_amanaged_disks_test.go @@ -22,11 +22,10 @@ package e2etest import ( "flag" + "github.com/Azure/azure-storage-azcopy/v10/common" "runtime" "testing" "time" - - "github.com/Azure/azure-storage-azcopy/v10/common" ) // Purpose: Tests for the special cases that relate to moving managed disks (default local VHD to page blob; special handling for From bc7c8ef03dd4cb723432073e014c9abc30637331 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 11:18:15 +0530 Subject: [PATCH 459/513] updated dependencies --- go.mod | 78 +++++++++++---------- go.sum | 215 ++++++++++++++++++++++++++++++++++----------------------- 2 files changed, 171 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index 60e1edee5..d2b3db4d9 100644 --- a/go.mod +++ b/go.mod @@ -1,61 +1,64 @@ module github.com/Azure/azure-storage-azcopy/v10 require ( - cloud.google.com/go/storage v1.29.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.1 - github.com/Azure/go-autorest/autorest/adal v0.9.18 + cloud.google.com/go/storage v1.43.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.3.0 + github.com/Azure/go-autorest/autorest/adal v0.9.24 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda - github.com/danieljoos/wincred v1.1.2 + github.com/danieljoos/wincred v1.2.2 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/uuid v1.6.0 - github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d - github.com/mattn/go-ieproxy v0.0.11 + github.com/hillu/go-ntdll v0.0.0-20240418173803-69345773b582 + github.com/mattn/go-ieproxy v0.0.12 github.com/minio/minio-go v6.0.14+incompatible github.com/pkg/errors v0.9.1 - github.com/pkg/xattr v0.4.6 + github.com/pkg/xattr v0.4.10 github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/spf13/cobra v1.4.0 + github.com/spf13/cobra v1.8.1 github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807 github.com/wastore/keyctl v0.3.1 - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/oauth2 v0.7.0 + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 - golang.org/x/sys v0.21.0 - google.golang.org/api v0.114.0 + golang.org/x/sys v0.22.0 + google.golang.org/api v0.189.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) require github.com/stretchr/testify v1.9.0 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 github.com/Azure/go-autorest/autorest/date v0.3.0 - golang.org/x/net v0.26.0 + golang.org/x/net v0.27.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect + cloud.google.com/go/iam v1.1.11 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-ini/ini v1.66.4 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -65,13 +68,18 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.56.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/genproto v0.0.0-20240723171418-e6d459c13d2a // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index cabcba293..e70b7db96 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,36 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1 h1:mkaGMgFkpDJVs7QUQrHvqEEpJFvoDrqGaHqMkywhGN0= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1/go.mod h1:3S0vo7Y+O3Fjnnon5JXVrlG2IrfQkXasvKWB4OwX1lk= -github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.1 h1:59Jk/lsLhtEMq6Zi7t6aWDFMman28m2O9K67IPAqqEI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.1/go.mod h1:b3FkvE9I9pO2DBaMOuml19tAgko7lPlKQPvhXnJXRoE= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +cloud.google.com/go/iam v1.1.11 h1:0mQ8UKSfdHLut6pH9FM3bI55KWR46ketn0PuXleDyxw= +cloud.google.com/go/iam v1.1.11/go.mod h1:biXoiLWYIKntto2joP+62sd9uW5EpkZmKIvfNcTWlnQ= +cloud.google.com/go/longrunning v0.5.10 h1:eB/BniENNRKhjz/xgiillrdcH3G74TGSl3BXinGlI7E= +cloud.google.com/go/longrunning v0.5.10/go.mod h1:tljz5guTr5oc/qhlUjBlk7UAIFMOGuPNxkNDZXlLics= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0 h1:gXpwp0sGZz2FY9lVpSdM1rMpsP9PUtevHQyFhGoqHxY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0/go.mod h1:K+OqH/n5xyCEvbenN5OtZMycqHRCeoHh0whMoRjWYK4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.3.0 h1:VIM+5hiNPIOmFy2Rax2VW+v7C8lbGEzgd25pnOBR0kY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.3.0/go.mod h1:amxdkUcI+5PQTRqJCPqWXf1CBMh/q3YGuFwyqpwAwd4= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -47,11 +49,11 @@ github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxB github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= +github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -59,9 +61,15 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/go-ini/ini v1.66.4 h1:dKjMqkcbkzfddhIhyglTPgMoJnkvmG+bSLrU9cTHc5M= -github.com/go-ini/ini v1.66.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -72,7 +80,6 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -81,31 +88,31 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= -github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d h1:WR9P3d5kJAzyQrNROnys5/wElv7ELkFCJ9t6gObxQbs= -github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d/go.mod h1:cHjYsnAnSckPDx8/H01Y+owD1hf2adLA6VRiw4guEbA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/hillu/go-ntdll v0.0.0-20240418173803-69345773b582 h1:msYGLGTxozPDUlpoVgYS6CrFw/PCRBOds5PGwo2XXuk= +github.com/hillu/go-ntdll v0.0.0-20240418173803-69345773b582/go.mod h1:cHjYsnAnSckPDx8/H01Y+owD1hf2adLA6VRiw4guEbA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -115,8 +122,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= -github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= +github.com/mattn/go-ieproxy v0.0.12 h1:OZkUFJC3ESNZPQ+6LzC3VJIFSnreeFLQyqvBWtvfL2M= +github.com/mattn/go-ieproxy v0.0.12/go.mod h1:Vn+N61199DAnVeTgaF8eoB9PvLO8P3OBnG95ENh7B7c= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -126,8 +133,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/xattr v0.4.6 h1:0vqthLIMxQKA9VscyMcxjvAUGvyfzlk009vwLE8OZJg= -github.com/pkg/xattr v0.4.6/go.mod h1:sBD3RAqlr8Q+RC3FutZcikpT8nyDrIEEBw2J744gVWs= +github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= +github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -136,8 +143,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -145,92 +152,130 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807 h1:Uzh85j0tl46Sf2OOx1wDePSWkz3Eq8XdCFkLXqaX8Bg= github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807/go.mod h1:zI8umr7xnBSyT9ZJ8wn48RiQ0EWXo4xmYLNw9FQvC9w= github.com/wastore/keyctl v0.3.1 h1:wMkYW9y9jGbQ1ARBLGLwnDdbgrkbuSeuIQeHy+BZOU0= github.com/wastore/keyctl v0.3.1/go.mod h1:1359RfMRDlblBSa2vaPC+kkmIxxt+rgl/FxLG38h9xM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20240723171418-e6d459c13d2a h1:hPbLwHFm59QoSKUT0uGaL19YN4U9W5lY4+iNXlUBNj0= +google.golang.org/genproto v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:+7gIV7FP6jBo5hiY2lsWA//NkNORQVj0J1Isc/4HzR4= +google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a h1:YIa/rzVqMEokBkPtydCkx1VLmv3An1Uw7w1P1m6EhOY= +google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a h1:hqK4+jJZXCU4pW7jsAdGOVFIfLHQeV7LaizZKnZ84HI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -240,15 +285,11 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 7c6ea26f2680730415da3e5169fad11a7f558a22 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 13:05:42 +0530 Subject: [PATCH 460/513] Version update --- common/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/version.go b/common/version.go index a8290327a..d6081f7a7 100644 --- a/common/version.go +++ b/common/version.go @@ -1,6 +1,6 @@ package common -const AzcopyVersion = "10.25.1" +const AzcopyVersion = "10.26.0" const UserAgent = "AzCopy/" + AzcopyVersion const S3ImportUserAgent = "S3Import " + UserAgent const GCPImportUserAgent = "GCPImport " + UserAgent From 7c14cc22bba7d39ebe561b3e1aeefa2eb29073d8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 20:37:30 +0530 Subject: [PATCH 461/513] test linux version --- release-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-pipeline.yml b/release-pipeline.yml index ec5be4bae..ea7cb85c6 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -99,10 +99,12 @@ stages: - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + $(root_dir)/azcopy_linux_amd64 --version displayName: 'Generate Linux AMD64' - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + $(root_dir)/azcopy_linux_se_amd64 --version displayName: 'Generate Linux AMD64 SE Integration' - script: | From e2dbf6d26f4b64546545b63eb6129ffa35d48fa8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 21:27:18 +0530 Subject: [PATCH 462/513] test linux version --- release-pipeline.yml | 1175 +++++++++++++++++++++--------------------- 1 file changed, 591 insertions(+), 584 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index ea7cb85c6..6aecd7748 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -147,6 +147,7 @@ stages: cp NOTICE.txt $(linux_amd64_dir)/ cp NOTICE.txt $(linux_se_amd64_dir)/ cp NOTICE.txt pkgDir/usr/bin/ + ./pkgDir/usr/bin/azcopy --version displayName: 'Copy required files for packaging' # using fpm tool for packaging of our binary & performing post-install operations @@ -206,334 +207,334 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(Build.SourcesDirectory)' - - name: archives - value: $(root_dir)/archives - - steps: - - checkout: self - displayName: 'Checkout repository' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_arm64: + # vmImage: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: 'blobfuse-ubn20-arm64' + + # pool: + # name: "blobfuse-ubn-arm64-pool" + # demands: + # - ImageOverride -equals $(agentName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(Build.SourcesDirectory)' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - checkout: self + # displayName: 'Checkout repository' + + # # Custom script to install Go-lang + # - task: ShellScript@2 + # inputs: + # scriptPath: "$(work_dir)/go_installer.sh" + # args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + # displayName: "Installing Go tools" + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Linux' + # workingDirectory: $(root_dir) + + # # install dependencies required for compiling azcopy + # - script: | + # sudo apt-get clean + # sudo apt-get update --fix-missing + # sudo apt-get install pkg-config libsecret-1-dev wget -y + # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + # sudo gem install dotenv -v 2.8.1 + # sudo gem install fpm -V + # displayName: "Installing Dependencies" + + # # get glibc version with which build is done + # - script: | + # ldd --version + # displayName: "GLIBC Version" + + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + # displayName: 'Generate Linux ARM64' - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(root_dir)/ - cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() + # - script: | + # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + # displayName: 'Generate Linux ARM64 SE Integration' + + # - script: | + # sudo ls -lRt $(root_dir)/ + # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + # displayName: 'List the binaries and add to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish binaries' + # condition: succeededOrFailed() - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + # displayName: "Remove binaries from staging directory" - - template: setup/azcopy_version_check_linux.yml - parameters: - azcopy_executable: $(root_dir)/azcopy_linux_arm64 - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' + # - template: setup/azcopy_version_check_linux.yml + # parameters: + # azcopy_executable: $(root_dir)/azcopy_linux_arm64 + + # - script: | + # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + # linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + # mkdir -p $linux_arm64_dir + # mkdir -p $linux_se_arm64_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mkdir -p pkgDir/usr/bin/ + # cp NOTICE.txt pkgDir/usr/bin/ + # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + # cp NOTICE.txt $(linux_arm64_dir)/ + # cp NOTICE.txt $(linux_se_arm64_dir)/ + # displayName: 'Copy required files for packaging' + + # # using fpm tool for packaging of our binary & performing post-install operations + # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + # - script: | + # fpm -s dir -t deb -n azcopy -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 deb Package' - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true + # - script: | + # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + # --maintainer "Azcopy v-Team " \ + # --url "https://github.com/Azure/azure-storage-azcopy" \ + # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Make ARM64 rpm Package' + + # - task: ArchiveFiles@2 + # displayName: 'Archive Standard Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Partner(SE) Linux Build' + # inputs: + # rootFolderOrFile: '$(linux_se_arm64_dir)' + # archiveType: tar + # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + # continueOnError: true - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + # workingDirectory: $(root_dir) + # displayName: 'Add tar.gz to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-linux-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -630,286 +631,286 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - - script: | - rm -rf azCopy-windows-temp/*.md - cp azCopy-windows-temp/* . - mv azCopy-windows-temp $(work_dir) - displayName: 'Add signed windows binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Artifacts' - - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - mv $(work_dir)/azCopy-windows-temp . - rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # - script: | + # rm -rf azCopy-windows-temp/*.md + # cp azCopy-windows-temp/* . + # mv azCopy-windows-temp $(work_dir) + # displayName: 'Add signed windows binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish Signed Artifacts' + + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # mv $(work_dir)/azCopy-windows-temp . + # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + # chmod 755 azCopy-windows-temp/* + # rm -rf azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # - script: | + # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir -p $windows_amd64 + # mkdir -p $windows_arm64 + # mkdir -p $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - rm -rf $(Build.ArtifactStagingDirectory)/*.md - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # rm -rf $(Build.ArtifactStagingDirectory)/*.md + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p unzipped_amd64 - mkdir -p unzipped_arm64 - unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - echo "AMD64 Folder name: $amd64FolderName" - - arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - # Check if there is exactly one matching directory for amd64 - if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching AMD64 directory: $amd64FolderName" - cd "$amd64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - else - echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - exit 1 - fi - - cd $(Build.ArtifactStagingDirectory) - - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching ARM64 directory: $arm64FolderName" - cd "$arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - else - echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - exit 1 - fi - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Mac Binary' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p unzipped_amd64 + # mkdir -p unzipped_arm64 + # unzip -o azcopy*amd64*.zip -d unzipped_amd64 + # unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + # echo "AMD64 Folder name: $amd64FolderName" + + # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + # echo "ARM64 Folder name: $arm64FolderName" + + # # Check if there is exactly one matching directory for amd64 + # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching AMD64 directory: $amd64FolderName" + # cd "$amd64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # else + # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + # exit 1 + # fi + + # cd $(Build.ArtifactStagingDirectory) + + # # Check if there is exactly one matching directory for arm64 + # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching ARM64 directory: $arm64FolderName" + # cd "$arm64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # else + # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + # exit 1 + # fi + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -965,6 +966,12 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | + # Locate the azcopy binary + which azcopy + + # Verify the binary + file $(which azcopy) + azcopy --version azcopy --help displayName: 'Check Version and Help' From b98b71c28ab4d33775ae02ef5fb4609e4237995a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 21:37:27 +0530 Subject: [PATCH 463/513] test linux version --- release-pipeline.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 6aecd7748..728d871bc 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1555,11 +1555,12 @@ stages: - name: Publish_To_Container value: ${{ parameters.publish_to_container }} - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - + value: '$(Build.SourcesDirectory)' steps: - + - checkout: self + displayName: 'Checkout repository' + - task: DownloadBuildArtifacts@0 displayName: 'Download Linux Build Artifacts' inputs: From 84c89752ffc1dbb62d8227c2837d8423dfb4fbe7 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 22:05:45 +0530 Subject: [PATCH 464/513] test linux version --- release-pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 728d871bc..5fd8e9295 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -959,9 +959,12 @@ stages: itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - script: | + ls -ltR azcopy-*x86_64.deb sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y + azcopy --version + azcopy --help displayName: 'Install deb Package' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed @@ -1560,7 +1563,7 @@ stages: steps: - checkout: self displayName: 'Checkout repository' - + - task: DownloadBuildArtifacts@0 displayName: 'Download Linux Build Artifacts' inputs: From 5d5348c86c77d713fc7686fb1a4248f56679e7e4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 22:31:39 +0530 Subject: [PATCH 465/513] test linux version 10.26.0 --- release-pipeline.yml | 1168 +++++++++++++++++++++--------------------- 1 file changed, 584 insertions(+), 584 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 5fd8e9295..bd266c929 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -207,334 +207,334 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_arm64: - # vmImage: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: 'blobfuse-ubn20-arm64' - - # pool: - # name: "blobfuse-ubn-arm64-pool" - # demands: - # - ImageOverride -equals $(agentName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(Build.SourcesDirectory)' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - checkout: self - # displayName: 'Checkout repository' - - # # Custom script to install Go-lang - # - task: ShellScript@2 - # inputs: - # scriptPath: "$(work_dir)/go_installer.sh" - # args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - # displayName: "Installing Go tools" - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Linux' - # workingDirectory: $(root_dir) - - # # install dependencies required for compiling azcopy - # - script: | - # sudo apt-get clean - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config libsecret-1-dev wget -y - # sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - # sudo gem install dotenv -v 2.8.1 - # sudo gem install fpm -V - # displayName: "Installing Dependencies" - - # # get glibc version with which build is done - # - script: | - # ldd --version - # displayName: "GLIBC Version" - - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - # displayName: 'Generate Linux ARM64' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(Build.SourcesDirectory)' + - name: archives + value: $(root_dir)/archives + + steps: + - checkout: self + displayName: 'Checkout repository' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' - # - script: | - # GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - # displayName: 'Generate Linux ARM64 SE Integration' - - # - script: | - # sudo ls -lRt $(root_dir)/ - # cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - # displayName: 'List the binaries and add to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish binaries' - # condition: succeededOrFailed() + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - # displayName: "Remove binaries from staging directory" + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" - # - template: setup/azcopy_version_check_linux.yml - # parameters: - # azcopy_executable: $(root_dir)/azcopy_linux_arm64 - - # - script: | - # linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - # linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - # mkdir -p $linux_arm64_dir - # mkdir -p $linux_se_arm64_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mkdir -p pkgDir/usr/bin/ - # cp NOTICE.txt pkgDir/usr/bin/ - # mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - # mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - # cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - # cp NOTICE.txt $(linux_arm64_dir)/ - # cp NOTICE.txt $(linux_se_arm64_dir)/ - # displayName: 'Copy required files for packaging' - - # # using fpm tool for packaging of our binary & performing post-install operations - # # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - # - script: | - # fpm -s dir -t deb -n azcopy -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - # cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 deb Package' + - template: setup/azcopy_version_check_linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_arm64 + + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' - # - script: | - # fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - # -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - # --maintainer "Azcopy v-Team " \ - # --url "https://github.com/Azure/azure-storage-azcopy" \ - # --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - # mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - # cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Make ARM64 rpm Package' - - # - task: ArchiveFiles@2 - # displayName: 'Archive Standard Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Partner(SE) Linux Build' - # inputs: - # rootFolderOrFile: '$(linux_se_arm64_dir)' - # archiveType: tar - # archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - # continueOnError: true + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true - # - script: | - # cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - # workingDirectory: $(root_dir) - # displayName: 'Add tar.gz to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-linux-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -631,286 +631,286 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # - script: | - # rm -rf azCopy-windows-temp/*.md - # cp azCopy-windows-temp/* . - # mv azCopy-windows-temp $(work_dir) - # displayName: 'Add signed windows binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Artifacts' - - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # mv $(work_dir)/azCopy-windows-temp . - # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - # chmod 755 azCopy-windows-temp/* - # rm -rf azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # - script: | - # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir -p $windows_amd64 - # mkdir -p $windows_arm64 - # mkdir -p $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + - script: | + rm -rf azCopy-windows-temp/*.md + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # rm -rf $(Build.ArtifactStagingDirectory)/*.md - # displayName: 'List Signed Artifacts' + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + rm -rf $(Build.ArtifactStagingDirectory)/*.md + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p unzipped_amd64 - # mkdir -p unzipped_arm64 - # unzip -o azcopy*amd64*.zip -d unzipped_amd64 - # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - # echo "AMD64 Folder name: $amd64FolderName" - - # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - # echo "ARM64 Folder name: $arm64FolderName" - - # # Check if there is exactly one matching directory for amd64 - # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching AMD64 directory: $amd64FolderName" - # cd "$amd64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # else - # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - # exit 1 - # fi - - # cd $(Build.ArtifactStagingDirectory) - - # # Check if there is exactly one matching directory for arm64 - # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching ARM64 directory: $arm64FolderName" - # cd "$arm64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # else - # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - # exit 1 - # fi - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Mac Binary' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' # SignArtifacts end here From c10025644835492be9064f0cd0b9f78ba42ac818 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 22:36:33 +0530 Subject: [PATCH 466/513] check verison --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6dedda333..4d17c33bd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -51,6 +51,7 @@ jobs: - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" + $(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64 --version displayName: 'Generate Linux AMD64 SE Integration' condition: eq(variables.type, 'linux') From 6646a40430c6d087fa730fecb6e792ff417f9741 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 24 Jul 2024 22:41:49 +0530 Subject: [PATCH 467/513] check with 20.04 ubuntu machine --- release-pipeline.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index bd266c929..5f638ead7 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -923,11 +923,14 @@ stages: strategy: matrix: Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(vmImage) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + steps: - checkout: none @@ -1544,11 +1547,13 @@ stages: strategy: matrix: Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" pool: - vmImage: $(vmImage) + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) variables: - name: Test_Release_Account_Name From 795d75448b6c953c113e16ccef56778b6ddc877c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 11:00:32 +0530 Subject: [PATCH 468/513] check with 20.04 ubuntu machine --- release-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipeline.yml b/release-pipeline.yml index 5f638ead7..befdf4a74 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -936,6 +936,7 @@ stages: - script: | sudo apt update + sudo apt-get install pkg-config libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" From 2c07eac6849b20c883c9708050f771850fcf977a Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 11:47:14 +0530 Subject: [PATCH 469/513] check with 22.04 ubuntu machine --- release-pipeline.yml | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index befdf4a74..169600a6c 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -920,16 +920,24 @@ stages: jobs: - job: Set_1 timeoutInMinutes: 120 + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) strategy: matrix: Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) + vmImage: $(vmImage) steps: - checkout: none @@ -1545,16 +1553,24 @@ stages: condition: succeeded('TestArtifacts') jobs: - job: ReleaseToContainer + # strategy: + # matrix: + # Ubuntu_amd64: + # imageName: 'ubuntu-20.04' + # container: 'test-cnt-ubn-20' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(agentName) strategy: matrix: Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) + vmImage: $(vmImage) variables: - name: Test_Release_Account_Name From 833fbb1f5700481b8276680afc9367e56d264601 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 11:48:18 +0530 Subject: [PATCH 470/513] check with 22.04 ubuntu machine --- release-pipeline.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/release-pipeline.yml b/release-pipeline.yml index 169600a6c..e09855225 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -961,6 +961,13 @@ stages: chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version + + # Locate the azcopy binary + which azcopy + + # Verify the binary + file $(which azcopy) + azcopy --version displayName: 'Test signed linux binaries' - task: DownloadBuildArtifacts@0 From 070a95aa0a46018a723403a8d6423e854c89077c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 12:16:18 +0530 Subject: [PATCH 471/513] check with 22.04 ubuntu machine --- release-pipeline.yml | 872 ++++++++++++++++++++++--------------------- 1 file changed, 441 insertions(+), 431 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index e09855225..07f65467b 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -367,174 +367,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows: + # imageName: 'windows-2019' + # type: 'windows' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Windows' + # workingDirectory: $(root_dir) + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + # displayName: 'Generate Windows AMD64' + # env: + # GOARCH: amd64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + # displayName: 'Generate Windows i386' + # env: + # GOARCH: 386 + # GOOS: windows + # CGO_ENABLED: 0 + + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm + # GOARM: 7 + # GOOS: windows + # CGO_ENABLED: 0 - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # - script: | + # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + # displayName: 'Generate Windows ARM' + # env: + # GOARCH: arm64 + # GOOS: windows + # CGO_ENABLED: 0 + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-13' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + + # - job: Set_4 + # timeoutInMinutes: 360 + # strategy: + # matrix: + # MacOS: + # imageName: 'macos-13' + # type: 'mac-os' + # pool: + # vmImage: $(imageName) + + # variables: + # - name: root_dir + # value: '$(System.DefaultWorkingDirectory)' + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: $(root_dir)/archives + + # steps: + # - task: GoTool@0 + # env: + # GO111MODULE: 'on' + # inputs: + # version: $(AZCOPY_GOLANG_VERSION) + + # - script: | + # echo 'Running GO Vet' + # go vet + # displayName: 'Golang Vet - Mac' + # workingDirectory: $(System.DefaultWorkingDirectory) + + # - script: | + # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + # displayName: 'Generate MacOS Build with AMD64' - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + # - script: | + # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + # displayName: 'Test Cross-compiled MacOS Build with ARM64' + + # - task: Bash@3 + # displayName: 'Extract AZCopy version' + # inputs: + # targetType: 'inline' + # script: | + # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + # - script: | + # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() + # mkdir -p $darwin_amd_dir + # mkdir -p $darwin_arm_dir + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # cp NOTICE.txt $(darwin_amd_dir) + # cp NOTICE.txt $(darwin_arm_dir) + # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + # displayName: 'Copy required files for packaging' + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_amd_dir)' + # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + # inputs: + # rootFolderOrFile: '$(darwin_arm_dir)' + # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-temp' + # displayName: 'Publish Artifacts' + # condition: succeededOrFailed() - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - # BuildArtifacts end here + # - task: ComponentGovernanceComponentDetection@0 + # inputs: + # scanType: 'Register' + # verbosity: 'Verbose' + # alertWarningLevel: 'High' + # # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -631,286 +631,286 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' + # - job: Set_2 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Windows-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + # - name: work_dir + # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + # - name: archives + # value: '$(work_dir)/archives' - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # steps: + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-windows-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) + + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download linux binary for extracting version' + # inputs: + # artifactName: 'azCopy-binaries' + # downloadPath: $(work_dir)/ + # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - - script: | - rm -rf azCopy-windows-temp/*.md - cp azCopy-windows-temp/* . - mv azCopy-windows-temp $(work_dir) - displayName: 'Add signed windows binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Artifacts' - - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - mv $(work_dir)/azCopy-windows-temp . - rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - displayName: 'Copy required files' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # sudo apt-get install -y zip + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_windows.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + # Pattern: 'azcopy_windows*' + + # - script: | + # rm -rf azCopy-windows-temp/*.md + # cp azCopy-windows-temp/* . + # mv azCopy-windows-temp $(work_dir) + # displayName: 'Add signed windows binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish Signed Artifacts' + + # # Validate signed images have md5sum changed + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/* + # mv $(work_dir)/azCopy-windows-temp . + # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + # chmod 755 azCopy-windows-temp/* + # rm -rf azCopy-windows-temp/*.md + # displayName: 'Make Artifacts executable' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # - script: | + # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # # Run azcopy --version and capture the output + # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + # if echo "$azcopyOutput" | grep -q "azcopy version"; then + # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + # if [[ -n "$version" ]]; then + # echo "Extracted version: $version" + # echo "##vso[task.setvariable variable=azcopy_version]$version" + # fi + # fi + # displayName: 'Extract the generated build version' + + # - script: | + # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + # echo "##vso[task.setvariable variable=windows_386]$windows_386" + + # mkdir -p $windows_amd64 + # mkdir -p $windows_arm64 + # mkdir -p $windows_386 + # mkdir -p $(archives) + # displayName: 'Create required directories' + + # - script: | + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + # cp NOTICE.txt $(windows_amd64) + # cp NOTICE.txt $(windows_arm64) + # cp NOTICE.txt $(windows_386) + # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + # displayName: 'Copy required files' - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - rm -rf $(Build.ArtifactStagingDirectory)/*.md - displayName: 'List Signed Artifacts' + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit AMD Build' + # inputs: + # rootFolderOrFile: '$(windows_amd64)' + # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 64 bit ARM Build' + # inputs: + # rootFolderOrFile: '$(windows_arm64)' + # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + # continueOnError: true + + # - task: ArchiveFiles@2 + # displayName: 'Archive Windows 32 bit Build' + # inputs: + # rootFolderOrFile: '$(windows_386)' + # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + # continueOnError: true + + # - script: | + # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + # displayName: 'Copy zip to staging directory' + + # - script: | + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # rm -rf $(Build.ArtifactStagingDirectory)/*.md + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_ESRP_SECRET - - steps: - - checkout: none + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-windows-signed' + # displayName: 'Publish Signed Artifacts' + + # - job: Set_3 + # timeoutInMinutes: 120 + # strategy: + # matrix: + # Mac-Signing: + # vmImage: 'Ubuntu-20.04' + # AgentName: "blobfuse-ubuntu20" + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: AZCOPY_ESRP_SECRET + + # steps: + # - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' + # - script: | + # echo ${{ parameters.tag }} + # displayName: 'Tag Name' - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) + # - task: DownloadBuildArtifacts@0 + # displayName: 'Download Build Artifacts' + # inputs: + # artifactName: 'azCopy-mac-temp' + # downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # displayName: 'List Artifacts' - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' + # - script: | + # sudo apt-get clean + # sudo apt-get update + # sudo apt-get upgrade -y + # sudo apt-get install unzip -y + # sudo apt-get install -f --fix-missing + # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + # sudo dpkg -i packages-microsoft-prod.deb + # sudo apt install apt-transport-https -y + # sudo apt install dotnet-sdk-6.0 -y + # displayName: "Update dependencies" + + # - template: setup/esrp_sign_notarize_macos.yml + # parameters: + # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + # Pattern: '*.zip' + + # # Validate signed images have md5sum changed + # - script: | + # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + # displayName: 'Make Artifacts executable' + + # - script: | + # sudo ls -lRt $(Build.ArtifactStagingDirectory) + # md5sum $(Build.ArtifactStagingDirectory)/* + # displayName: 'List Signed Artifacts' - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p unzipped_amd64 - mkdir -p unzipped_arm64 - unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - echo "AMD64 Folder name: $amd64FolderName" - - arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - # Check if there is exactly one matching directory for amd64 - if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching AMD64 directory: $amd64FolderName" - cd "$amd64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - else - echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - exit 1 - fi - - cd $(Build.ArtifactStagingDirectory) - - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching ARM64 directory: $arm64FolderName" - cd "$arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - else - echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - exit 1 - fi - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Mac Binary' + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-mac-signed' + # displayName: 'Publish Signed Artifacts' + + # - script: | + # mkdir -p unzipped_amd64 + # mkdir -p unzipped_arm64 + # unzip -o azcopy*amd64*.zip -d unzipped_amd64 + # unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # # Use find to retrieve the folder name + # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + # echo "AMD64 Folder name: $amd64FolderName" + + # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + # echo "ARM64 Folder name: $arm64FolderName" + + # # Check if there is exactly one matching directory for amd64 + # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching AMD64 directory: $amd64FolderName" + # cd "$amd64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + # else + # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + # exit 1 + # fi + + # cd $(Build.ArtifactStagingDirectory) + + # # Check if there is exactly one matching directory for arm64 + # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + # echo "Found matching ARM64 directory: $arm64FolderName" + # cd "$arm64FolderName" + # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + # else + # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + # exit 1 + # fi + + # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + # rm -rf $(Build.ArtifactStagingDirectory)/*.zip + # sudo ls -ltR $(Build.ArtifactStagingDirectory) + # displayName: 'Copy and list signed mac binary to staging directory' + # workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # # Push signed images to artifact directory + # - task: PublishBuildArtifacts@1 + # inputs: + # artifactName: 'azCopy-binaries' + # displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -968,6 +968,10 @@ stages: # Verify the binary file $(which azcopy) azcopy --version + + ls /usr/local/bin/azcopy + ls /usr/bin/azcopy + sudo find / -name azcopy -type f -exec rm -f {} \; displayName: 'Test signed linux binaries' - task: DownloadBuildArtifacts@0 @@ -988,11 +992,17 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | + + # Locate the azcopy binary which azcopy # Verify the binary file $(which azcopy) + azcopy --version + + ls /usr/local/bin/azcopy + ls /usr/bin/azcopy azcopy --version azcopy --help From f4ec93ed7ea43dcdb9c17b4538ae4336dd06cf23 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 12:48:58 +0530 Subject: [PATCH 472/513] check with 22.04 ubuntu machine azcopy version --- release-pipeline.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 07f65467b..c8415a11a 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -941,7 +941,20 @@ stages: steps: - checkout: none - + + #TODO remove + - script: | + # Locate the azcopy binary + which azcopy + + # Verify the binary + file $(which azcopy) + azcopy --version + + ls /usr/local/bin/azcopy + ls /usr/bin/azcopy + displayName: 'Check azcopy version if any' + - script: | sudo apt update sudo apt-get install pkg-config libsecret-1-dev -y From 0e56a72b90ac02b19cc8793b32c6db46c99e815f Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 12:59:11 +0530 Subject: [PATCH 473/513] pipeline version failure fix --- azure-pipelines.yml | 1 - release-pipeline.yml | 911 +++++++++++++++++++++---------------------- 2 files changed, 434 insertions(+), 478 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4d17c33bd..6dedda333 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -51,7 +51,6 @@ jobs: - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64" - $(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64 --version displayName: 'Generate Linux AMD64 SE Integration' condition: eq(variables.type, 'linux') diff --git a/release-pipeline.yml b/release-pipeline.yml index c8415a11a..807f3da69 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -367,174 +367,174 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows: - # imageName: 'windows-2019' - # type: 'windows' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Windows' - # workingDirectory: $(root_dir) - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - # displayName: 'Generate Windows AMD64' - # env: - # GOARCH: amd64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - # displayName: 'Generate Windows i386' - # env: - # GOARCH: 386 - # GOOS: windows - # CGO_ENABLED: 0 - - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm - # GOARM: 7 - # GOOS: windows - # CGO_ENABLED: 0 + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 - # - script: | - # go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - # displayName: 'Generate Windows ARM' - # env: - # GOARCH: arm64 - # GOOS: windows - # CGO_ENABLED: 0 - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - - # - job: Set_4 - # timeoutInMinutes: 360 - # strategy: - # matrix: - # MacOS: - # imageName: 'macos-13' - # type: 'mac-os' - # pool: - # vmImage: $(imageName) - - # variables: - # - name: root_dir - # value: '$(System.DefaultWorkingDirectory)' - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: $(root_dir)/archives - - # steps: - # - task: GoTool@0 - # env: - # GO111MODULE: 'on' - # inputs: - # version: $(AZCOPY_GOLANG_VERSION) - - # - script: | - # echo 'Running GO Vet' - # go vet - # displayName: 'Golang Vet - Mac' - # workingDirectory: $(System.DefaultWorkingDirectory) - - # - script: | - # CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - # displayName: 'Generate MacOS Build with AMD64' + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-13' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' - # - script: | - # GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - # displayName: 'Test Cross-compiled MacOS Build with ARM64' - - # - task: Bash@3 - # displayName: 'Extract AZCopy version' - # inputs: - # targetType: 'inline' - # script: | - # azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - # echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - # - script: | - # darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - # darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - # mkdir -p $darwin_amd_dir - # mkdir -p $darwin_arm_dir - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # cp NOTICE.txt $(darwin_amd_dir) - # cp NOTICE.txt $(darwin_arm_dir) - # mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - # mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - # displayName: 'Copy required files for packaging' - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_amd_dir)' - # archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - # inputs: - # rootFolderOrFile: '$(darwin_arm_dir)' - # archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-temp' - # displayName: 'Publish Artifacts' - # condition: succeededOrFailed() + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() - # - task: ComponentGovernanceComponentDetection@0 - # inputs: - # scanType: 'Register' - # verbosity: 'Verbose' - # alertWarningLevel: 'High' - # # BuildArtifacts end here + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts dependsOn: BuildArtifacts @@ -631,286 +631,286 @@ stages: artifactName: 'azCopy-linux-signed' displayName: 'Publish Signed Artifacts' - # - job: Set_2 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Windows-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - # - name: work_dir - # value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - # - name: archives - # value: '$(work_dir)/archives' + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' - # steps: - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-windows-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) - - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download linux binary for extracting version' - # inputs: - # artifactName: 'azCopy-binaries' - # downloadPath: $(work_dir)/ - # itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # sudo apt-get install -y zip - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_windows.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - # Pattern: 'azcopy_windows*' - - # - script: | - # rm -rf azCopy-windows-temp/*.md - # cp azCopy-windows-temp/* . - # mv azCopy-windows-temp $(work_dir) - # displayName: 'Add signed windows binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Artifacts' - - # # Validate signed images have md5sum changed - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/* - # mv $(work_dir)/azCopy-windows-temp . - # rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - # chmod 755 azCopy-windows-temp/* - # rm -rf azCopy-windows-temp/*.md - # displayName: 'Make Artifacts executable' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # - script: | - # chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # # Run azcopy --version and capture the output - # azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - # if echo "$azcopyOutput" | grep -q "azcopy version"; then - # version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - # if [[ -n "$version" ]]; then - # echo "Extracted version: $version" - # echo "##vso[task.setvariable variable=azcopy_version]$version" - # fi - # fi - # displayName: 'Extract the generated build version' - - # - script: | - # windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - # windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - # windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - # echo "##vso[task.setvariable variable=windows_386]$windows_386" - - # mkdir -p $windows_amd64 - # mkdir -p $windows_arm64 - # mkdir -p $windows_386 - # mkdir -p $(archives) - # displayName: 'Create required directories' - - # - script: | - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - # mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - # cp NOTICE.txt $(windows_amd64) - # cp NOTICE.txt $(windows_arm64) - # cp NOTICE.txt $(windows_386) - # zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - # zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - # displayName: 'Copy required files' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + - script: | + rm -rf azCopy-windows-temp/*.md + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + displayName: 'Copy required files' - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit AMD Build' - # inputs: - # rootFolderOrFile: '$(windows_amd64)' - # archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 64 bit ARM Build' - # inputs: - # rootFolderOrFile: '$(windows_arm64)' - # archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - # continueOnError: true - - # - task: ArchiveFiles@2 - # displayName: 'Archive Windows 32 bit Build' - # inputs: - # rootFolderOrFile: '$(windows_386)' - # archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - # continueOnError: true - - # - script: | - # cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - # displayName: 'Copy zip to staging directory' - - # - script: | - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # rm -rf $(Build.ArtifactStagingDirectory)/*.md - # displayName: 'List Signed Artifacts' + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + rm -rf $(Build.ArtifactStagingDirectory)/*.md + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-windows-signed' - # displayName: 'Publish Signed Artifacts' - - # - job: Set_3 - # timeoutInMinutes: 120 - # strategy: - # matrix: - # Mac-Signing: - # vmImage: 'Ubuntu-20.04' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: AZCOPY_ESRP_SECRET - - # steps: - # - checkout: none + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_ESRP_SECRET + + steps: + - checkout: none - # - script: | - # echo ${{ parameters.tag }} - # displayName: 'Tag Name' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' - # - task: DownloadBuildArtifacts@0 - # displayName: 'Download Build Artifacts' - # inputs: - # artifactName: 'azCopy-mac-temp' - # downloadPath: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # displayName: 'List Artifacts' + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' - # - script: | - # sudo apt-get clean - # sudo apt-get update - # sudo apt-get upgrade -y - # sudo apt-get install unzip -y - # sudo apt-get install -f --fix-missing - # wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - # sudo dpkg -i packages-microsoft-prod.deb - # sudo apt install apt-transport-https -y - # sudo apt install dotnet-sdk-6.0 -y - # displayName: "Update dependencies" - - # - template: setup/esrp_sign_notarize_macos.yml - # parameters: - # FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - # ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - # Pattern: '*.zip' - - # # Validate signed images have md5sum changed - # - script: | - # chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - # mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - # rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - # displayName: 'Make Artifacts executable' - - # - script: | - # sudo ls -lRt $(Build.ArtifactStagingDirectory) - # md5sum $(Build.ArtifactStagingDirectory)/* - # displayName: 'List Signed Artifacts' + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-mac-signed' - # displayName: 'Publish Signed Artifacts' - - # - script: | - # mkdir -p unzipped_amd64 - # mkdir -p unzipped_arm64 - # unzip -o azcopy*amd64*.zip -d unzipped_amd64 - # unzip -o azcopy*arm64*.zip -d unzipped_arm64 - - # # Use find to retrieve the folder name - # amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - # echo "AMD64 Folder name: $amd64FolderName" - - # arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - # echo "ARM64 Folder name: $arm64FolderName" - - # # Check if there is exactly one matching directory for amd64 - # if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching AMD64 directory: $amd64FolderName" - # cd "$amd64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - # else - # echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - # exit 1 - # fi - - # cd $(Build.ArtifactStagingDirectory) - - # # Check if there is exactly one matching directory for arm64 - # if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - # echo "Found matching ARM64 directory: $arm64FolderName" - # cd "$arm64FolderName" - # mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - # else - # echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - # exit 1 - # fi - - # rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - # rm -rf $(Build.ArtifactStagingDirectory)/*.zip - # sudo ls -ltR $(Build.ArtifactStagingDirectory) - # displayName: 'Copy and list signed mac binary to staging directory' - # workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # # Push signed images to artifact directory - # - task: PublishBuildArtifacts@1 - # inputs: - # artifactName: 'azCopy-binaries' - # displayName: 'Publish Signed Mac Binary' + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' # SignArtifacts end here @@ -920,16 +920,6 @@ stages: jobs: - job: Set_1 timeoutInMinutes: 120 - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) strategy: matrix: Ubuntu_amd64: @@ -954,7 +944,7 @@ stages: ls /usr/local/bin/azcopy ls /usr/bin/azcopy displayName: 'Check azcopy version if any' - + - script: | sudo apt update sudo apt-get install pkg-config libsecret-1-dev -y @@ -974,17 +964,7 @@ stages: chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version - - # Locate the azcopy binary - which azcopy - - # Verify the binary - file $(which azcopy) - azcopy --version - - ls /usr/local/bin/azcopy - ls /usr/bin/azcopy - sudo find / -name azcopy -type f -exec rm -f {} \; + rm -rf /usr/local/bin/azcopy displayName: 'Test signed linux binaries' - task: DownloadBuildArtifacts@0 @@ -999,24 +979,10 @@ stages: sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y - azcopy --version - azcopy --help displayName: 'Install deb Package' workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - script: | - - - # Locate the azcopy binary - which azcopy - - # Verify the binary - file $(which azcopy) - azcopy --version - - ls /usr/local/bin/azcopy - ls /usr/bin/azcopy - azcopy --version azcopy --help displayName: 'Check Version and Help' @@ -1583,16 +1549,6 @@ stages: condition: succeeded('TestArtifacts') jobs: - job: ReleaseToContainer - # strategy: - # matrix: - # Ubuntu_amd64: - # imageName: 'ubuntu-20.04' - # container: 'test-cnt-ubn-20' - # AgentName: "blobfuse-ubuntu20" - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(agentName) strategy: matrix: Ubuntu_amd64: @@ -1643,6 +1599,7 @@ stages: - script: | sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + rm -rf /usr/local/bin/azcopy sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y From f5f5ea24fdce0bd736f8c338cf8a2f6568925d47 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 13:06:32 +0530 Subject: [PATCH 474/513] version fix for deb package --- release-pipeline.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 807f3da69..3dc9a766f 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -99,12 +99,10 @@ stages: - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - $(root_dir)/azcopy_linux_amd64 --version displayName: 'Generate Linux AMD64' - script: | GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - $(root_dir)/azcopy_linux_se_amd64 --version displayName: 'Generate Linux AMD64 SE Integration' - script: | @@ -147,7 +145,6 @@ stages: cp NOTICE.txt $(linux_amd64_dir)/ cp NOTICE.txt $(linux_se_amd64_dir)/ cp NOTICE.txt pkgDir/usr/bin/ - ./pkgDir/usr/bin/azcopy --version displayName: 'Copy required files for packaging' # using fpm tool for packaging of our binary & performing post-install operations @@ -928,26 +925,11 @@ stages: container: 'test-cnt-ubn-22' pool: vmImage: $(vmImage) - steps: - checkout: none - #TODO remove - - script: | - # Locate the azcopy binary - which azcopy - - # Verify the binary - file $(which azcopy) - azcopy --version - - ls /usr/local/bin/azcopy - ls /usr/bin/azcopy - displayName: 'Check azcopy version if any' - - script: | sudo apt update - sudo apt-get install pkg-config libsecret-1-dev -y sudo apt --fix-broken install ldd --version displayName: "GLIBC Version" @@ -975,7 +957,6 @@ stages: itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - script: | - ls -ltR azcopy-*x86_64.deb sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y From 5122d10a2fd3d3e3968b3654fbfb33aa8c0fbf61 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 13:19:28 +0530 Subject: [PATCH 475/513] change log type for github release --- release-pipeline.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 3dc9a766f..f0caa3e5e 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1651,8 +1651,6 @@ stages: - ImageOverride -equals $(AgentName) steps: - - checkout: none - - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' @@ -1713,7 +1711,7 @@ stages: $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' + changeLogFile: '$(Build.SourcesDirectory)/ChangeLog.md' isDraft: ${{ parameters.draft }} isPreRelease: ${{ parameters.prerelease }} assetUploadMode: replace From b6444c3ddc4951dd2f276191071869c049cd7a9e Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 15:28:47 +0530 Subject: [PATCH 476/513] added v10.26.0 release details --- ChangeLog.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index eab094c9f..9a6c412da 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,22 @@ # Change Log +## Version 10.26.0 + +### Security fixes + +1. Updated dependencies to address security vulnerabilities. + +## Bug Fixes + +1. Fixed an issue where AzCopy would fail to unmarshal the `_token_refresh_source` property correctly when performing copy jobs from OAuth-attached containers. ([#2710](https://github.com/Azure/azure-storage-azcopy/pull/2710)) +#TBD: is this required to be added in bug-fix section as its a CI pipeline change?? Check on this +2. Fixed a CI pipeline in Azure DevOps to automatically detect CVEs declared against our dependencies. ([#2705](https://github.com/Azure/azure-storage-azcopy/pull/2705)) + +### Dependency updates + +1. Golang 1.22.4 -> 1.22.5 + ## Version 10.25.1 ### Security fixes From 12c14baf7ea1eae6a7ef1db45c2d77abc01fb640 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Thu, 25 Jul 2024 15:51:15 +0530 Subject: [PATCH 477/513] incorporated review comments --- ChangeLog.md | 8 ++++---- release-pipeline.yml | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 9a6c412da..9c9e5ab65 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -7,16 +7,16 @@ 1. Updated dependencies to address security vulnerabilities. +### Dependency updates + +1. Golang 1.22.4 -> 1.22.5 + ## Bug Fixes 1. Fixed an issue where AzCopy would fail to unmarshal the `_token_refresh_source` property correctly when performing copy jobs from OAuth-attached containers. ([#2710](https://github.com/Azure/azure-storage-azcopy/pull/2710)) #TBD: is this required to be added in bug-fix section as its a CI pipeline change?? Check on this 2. Fixed a CI pipeline in Azure DevOps to automatically detect CVEs declared against our dependencies. ([#2705](https://github.com/Azure/azure-storage-azcopy/pull/2705)) -### Dependency updates - -1. Golang 1.22.4 -> 1.22.5 - ## Version 10.25.1 ### Security fixes diff --git a/release-pipeline.yml b/release-pipeline.yml index f0caa3e5e..c78ab07f9 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -947,6 +947,7 @@ stages: $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version rm -rf /usr/local/bin/azcopy + rm -rf /usr/bin/azcopy displayName: 'Test signed linux binaries' - task: DownloadBuildArtifacts@0 @@ -1581,6 +1582,7 @@ stages: sudo apt-get install -y zip cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . rm -rf /usr/local/bin/azcopy + rm -rf /usr/bin/azcopy sudo dpkg --info azcopy-*x86_64.deb sudo dpkg -i azcopy-*x86_64.deb sudo apt-get install build-essential -y From d95ed5312981485f2e30606917e53e4f14abd672 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 11:52:25 +0530 Subject: [PATCH 478/513] incorporated review comments --- ChangeLog.md | 6 +++++- release-pipeline.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 9c9e5ab65..ecc92ff39 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -7,14 +7,18 @@ 1. Updated dependencies to address security vulnerabilities. +### New Features + +1. AzCopy now supports distribution through package managers for Red Hat Enterprise Linux (RHEL), Ubuntu,Mariner, Debian, SUSE, Rocky, and CentOS. ([#2728](https://github.com/Azure/azure-storage-azcopy/pull/2728)) + ### Dependency updates 1. Golang 1.22.4 -> 1.22.5 +2. azidentity 1.6.0 -> 1.7.0 ## Bug Fixes 1. Fixed an issue where AzCopy would fail to unmarshal the `_token_refresh_source` property correctly when performing copy jobs from OAuth-attached containers. ([#2710](https://github.com/Azure/azure-storage-azcopy/pull/2710)) -#TBD: is this required to be added in bug-fix section as its a CI pipeline change?? Check on this 2. Fixed a CI pipeline in Azure DevOps to automatically detect CVEs declared against our dependencies. ([#2705](https://github.com/Azure/azure-storage-azcopy/pull/2705)) ## Version 10.25.1 diff --git a/release-pipeline.yml b/release-pipeline.yml index c78ab07f9..9132a74bb 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1713,7 +1713,7 @@ stages: $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* changeLogCompareToRelease: 'lastFullRelease' - changeLogFile: '$(Build.SourcesDirectory)/ChangeLog.md' + changeLogFile: 'commitBased' isDraft: ${{ parameters.draft }} isPreRelease: ${{ parameters.prerelease }} assetUploadMode: replace From 8c055e1683a4a477cb775644c327fdf49b578fad Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 11:53:48 +0530 Subject: [PATCH 479/513] incorporated review comments --- ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index ecc92ff39..fa7876767 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -9,7 +9,7 @@ ### New Features -1. AzCopy now supports distribution through package managers for Red Hat Enterprise Linux (RHEL), Ubuntu,Mariner, Debian, SUSE, Rocky, and CentOS. ([#2728](https://github.com/Azure/azure-storage-azcopy/pull/2728)) +1. AzCopy now supports distribution through package managers for Red Hat Enterprise Linux (RHEL), Ubuntu, Mariner, Debian, SUSE, Rocky and CentOS. ([#2728](https://github.com/Azure/azure-storage-azcopy/pull/2728)) ### Dependency updates From 1b3f6a70c2a8524a55f22197e9610aa92b287ff8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 12:11:22 +0530 Subject: [PATCH 480/513] incorporated review comments --- release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 9132a74bb..0ac42a391 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1713,7 +1713,7 @@ stages: $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* changeLogCompareToRelease: 'lastFullRelease' - changeLogFile: 'commitBased' + changeLogType: 'commitBased' isDraft: ${{ parameters.draft }} isPreRelease: ${{ parameters.prerelease }} assetUploadMode: replace From 3eb97e5c0d94b6b2123f98e4eb9557265edc91b2 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 13:29:09 +0530 Subject: [PATCH 481/513] Added support for M1 build release --- release-pipeline.yml | 190 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 184 insertions(+), 6 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 0ac42a391..f72fb6bef 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -12,6 +12,11 @@ parameters: type: boolean default: false + - name: publish_m1_build_to_container + displayName: 'Publish M1 build To Container' + type: boolean + default: false + - name: post_release displayName: 'Post Release on Github' type: boolean @@ -441,7 +446,7 @@ stages: strategy: matrix: MacOS: - imageName: 'macos-13' + imageName: 'macos-latest' type: 'mac-os' pool: vmImage: $(imageName) @@ -531,6 +536,86 @@ stages: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS M1: + imageName: 'macos-14' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" + displayName: 'MacOS M1 Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_m1_arm_dir="$(work_dir)/azcopy_darwin_m1_arm64$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_m1_arm_dir]$darwin_m1_arm_dir" + + mkdir -p $darwin_m1_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_m1_arm_dir) + cp $(root_dir)/azcopy $(darwin_m1_arm_dir) + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS M1 ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_m1_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_m1_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + # BuildArtifacts end here - stage: SignArtifacts @@ -865,8 +950,10 @@ stages: - script: | mkdir -p unzipped_amd64 mkdir -p unzipped_arm64 + mkdir -p unzipped_m1_arm64 unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy*arm64*.zip -d unzipped_arm64 + unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64 + unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64 # Use find to retrieve the folder name amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") @@ -875,6 +962,9 @@ stages: arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") echo "ARM64 Folder name: $arm64FolderName" + m1Arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*") + echo "M1 ARM64 Folder name: $m1Arm64FolderName" + # Check if there is exactly one matching directory for amd64 if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then echo "Found matching AMD64 directory: $amd64FolderName" @@ -897,6 +987,17 @@ stages: exit 1 fi + cd $(Build.ArtifactStagingDirectory) + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$m1Arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching M1 ARM64 directory: $m1Arm64FolderName" + cd "$m1Arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_m1_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$m1Arm64FolderName" | wc -l)." + exit 1 + fi + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) @@ -1353,7 +1454,7 @@ stages: strategy: matrix: MacOS: - imageName: 'macos-13' + imageName: 'macos-latest' type: 'mac-os' pool: vmImage: $(imageName) @@ -1418,6 +1519,77 @@ stages: ./azcopy --version ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + + - job: Set_9 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS M1: + imageName: 'macos-14' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy_darwin_m1_arm64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy_darwin_m1_arm64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_m1_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - job: Set_9 timeoutInMinutes: 120 @@ -1551,9 +1723,6 @@ stages: value: '$(Build.SourcesDirectory)' steps: - - checkout: self - displayName: 'Checkout repository' - - task: DownloadBuildArtifacts@0 displayName: 'Download Linux Build Artifacts' inputs: @@ -1640,6 +1809,12 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + if [ -z "$M1_CONTAINER_URL" ]; then + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$M1_CONTAINER_URL" --put-md5=true + else + echo "Azcopy M1 build not uploaded as M1_CONTAINER_URL variable is not set" + fi + - job: ReleaseAzcopy timeoutInMinutes: 120 strategy: @@ -1653,6 +1828,8 @@ stages: - ImageOverride -equals $(AgentName) steps: + - checkout: none + - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' @@ -1868,6 +2045,7 @@ stages: displayName: 'Publish the repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ +- ${{ if eq(parameters.update_version, true) }}: - stage: UpdateLatestVersion dependsOn: ReleaseArtifacts condition: succeeded('ReleaseArtifacts') From c583b5c67aa5ba0c96c9ac0cdc38dee8db70bb51 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 13:35:22 +0530 Subject: [PATCH 482/513] Added support for M1 build release --- release-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index f72fb6bef..2f872c1c6 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1520,7 +1520,7 @@ stages: ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_9 + - job: Set_9 timeoutInMinutes: 360 strategy: matrix: @@ -1829,7 +1829,7 @@ stages: steps: - checkout: none - + - script: | echo ${{ parameters.tag }} displayName: 'Tag Name' From 6abcc0a0a75f372aa499db95c03d1adb3aadac88 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 13:37:25 +0530 Subject: [PATCH 483/513] Added support for M1 build release --- release-pipeline.yml | 94 ++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 2f872c1c6..b9d34f2d9 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -2045,51 +2045,51 @@ stages: displayName: 'Publish the repository' workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ -- ${{ if eq(parameters.update_version, true) }}: - - stage: UpdateLatestVersion - dependsOn: ReleaseArtifacts - condition: succeeded('ReleaseArtifacts') - jobs: - - job: UpdateVersion - pool: - vmImage: 'ubuntu-20.04' - variables: - - group: AZCOPY_TESTS_VAR - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - checkout: none - - - ${{ if eq(parameters.update_version, true) }}: - - script: | - sudo apt-get install python3 -y - python3 --version - displayName: 'Installing Python' + - ${{ if eq(parameters.update_version, true) }}: + - stage: UpdateLatestVersion + dependsOn: ReleaseArtifacts + condition: succeeded('ReleaseArtifacts') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - # install azcopy - - script: | - cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - ls azcopy*.arm64.deb - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - azcopy --version - displayName: 'Installing azcopy' - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py - ls -l - displayName: 'Getting Python script' - workingDirectory: $(root_dir) - - - script: | - pip install azure-storage-blob - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" - displayName: 'Updating version number' - workingDirectory: $(root_dir) + - ${{ if eq(parameters.update_version, true) }}: + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) From 261987c385d1053c6b5f1ac6dd1ac859c9b652f1 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 13:42:39 +0530 Subject: [PATCH 484/513] Added support for M1 build release --- release-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index b9d34f2d9..ac68c947c 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -537,7 +537,7 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_4 + - job: Set_5 timeoutInMinutes: 360 strategy: matrix: @@ -1591,7 +1591,7 @@ stages: ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_9 + - job: Set_10 timeoutInMinutes: 120 strategy: matrix: From 22bf19ab6888524572c2c89f8f4d17814a55c86d Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 15:05:36 +0530 Subject: [PATCH 485/513] Added support for M1 build release --- release-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index ac68c947c..1f39ef709 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -953,7 +953,7 @@ stages: mkdir -p unzipped_m1_arm64 unzip -o azcopy*amd64*.zip -d unzipped_amd64 unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64 - unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64 + unzip -o azcopy_darwin_m1_arm64_*.zip -d unzipped_m1_arm64 # Use find to retrieve the folder name amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") @@ -962,7 +962,7 @@ stages: arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") echo "ARM64 Folder name: $arm64FolderName" - m1Arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*") + m1Arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64_*") echo "M1 ARM64 Folder name: $m1Arm64FolderName" # Check if there is exactly one matching directory for amd64 @@ -975,7 +975,7 @@ stages: exit 1 fi - cd $(Build.ArtifactStagingDirectory) + cd $(Build.ArtifactStagingDirectory) # Check if there is exactly one matching directory for arm64 if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then @@ -1810,7 +1810,7 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true if [ -z "$M1_CONTAINER_URL" ]; then - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$M1_CONTAINER_URL" --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" $(M1_CONTAINER_URL) --put-md5=true else echo "Azcopy M1 build not uploaded as M1_CONTAINER_URL variable is not set" fi From d4bb732fd9812c8a63924e52a3dea3b5302cb69c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 15:15:38 +0530 Subject: [PATCH 486/513] Added support for M1 build release --- release-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 1f39ef709..5aa86398b 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1719,6 +1719,8 @@ stages: value: 'azcopyvnextrelease' - name: Publish_To_Container value: ${{ parameters.publish_to_container }} + - name: Publish_To_M1_Container + value: ${{ parameters.publish_to_m1_container }} - name: work_dir value: '$(Build.SourcesDirectory)' @@ -1809,10 +1811,8 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - if [ -z "$M1_CONTAINER_URL" ]; then - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" $(M1_CONTAINER_URL) --put-md5=true - else - echo "Azcopy M1 build not uploaded as M1_CONTAINER_URL variable is not set" + if [ $(Publish_To_M1_Container) = "True" ]; then + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$(M1_CONTAINER_URL)" --put-md5=true fi - job: ReleaseAzcopy From 8daee2ae03838289bcea8ab6954d0b1ff3f9691c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 15:16:58 +0530 Subject: [PATCH 487/513] Added support for M1 build release --- release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 5aa86398b..794019840 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1720,7 +1720,7 @@ stages: - name: Publish_To_Container value: ${{ parameters.publish_to_container }} - name: Publish_To_M1_Container - value: ${{ parameters.publish_to_m1_container }} + value: ${{ parameters.publish_m1_build_to_container }} - name: work_dir value: '$(Build.SourcesDirectory)' From 97fdd918040bf2938e6a947b510e517f3e017c34 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 15:25:47 +0530 Subject: [PATCH 488/513] Added support for M1 build release --- release-pipeline.yml | 152 ++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 74 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 794019840..59eb21ce0 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1814,86 +1814,90 @@ stages: if [ $(Publish_To_M1_Container) = "True" ]; then AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$(M1_CONTAINER_URL)" --put-md5=true fi - - - job: ReleaseAzcopy - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - steps: - - checkout: none - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + - ${{ if eq(parameters.post_release, true) }}: + - stage: ReleaseArtifacts + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToGithub + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) + steps: + - checkout: none - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm - md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip - displayName: 'List Artifacts' + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) - - ${{ if eq(parameters.post_release, true) }}: - - - task: GithubRelease@1 - inputs: - githubConnection: 'azcopy-github-connection' - repositoryName: 'Azure/azure-storage-azcopy' - action: 'edit' - - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' - - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} - - assets: | - $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* - $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.deb + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.rpm + md5sum $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*.tar.gz + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*.zip + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*.zip + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - task: GithubRelease@1 + inputs: + githubConnection: 'azcopy-github-connection' + repositoryName: 'Azure/azure-storage-azcopy' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + assets: | + $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/* + $(Build.ArtifactStagingDirectory)/azCopy-windows-signed/* + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace - ${{ if eq(parameters.publish_artifacts, true) }}: - stage: PublishArtifacts From 2d2f8e19628798939d63e3000243a8c6b9feb10b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 15:27:20 +0530 Subject: [PATCH 489/513] Added support for M1 build release --- release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 59eb21ce0..cd9a0f57a 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1816,7 +1816,7 @@ stages: fi - - ${{ if eq(parameters.post_release, true) }}: + - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseArtifacts dependsOn: TestArtifacts condition: succeeded('TestArtifacts') From 7c8dd3fcfb3ff4cc605d8831fea767f031821dcb Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 15:59:33 +0530 Subject: [PATCH 490/513] Added support for M1 build release --- release-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-pipeline.yml b/release-pipeline.yml index cd9a0f57a..edadac07a 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1713,6 +1713,7 @@ stages: vmImage: $(vmImage) variables: + - group: AZCOPY_TESTS_VAR - name: Test_Release_Account_Name value: 'azcopyprivatedrops' - name: Release_Account_Name From 9647c33a5babcc59719f5d64b8a6b898cc626671 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 16:48:20 +0530 Subject: [PATCH 491/513] Added support for M1 build release --- release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index edadac07a..b26c1d44d 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -962,7 +962,7 @@ stages: arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") echo "ARM64 Folder name: $arm64FolderName" - m1Arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64_*") + m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64_*") echo "M1 ARM64 Folder name: $m1Arm64FolderName" # Check if there is exactly one matching directory for amd64 From 8201b08c86cdb215aaa194fede795f4332ebe282 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 17:22:49 +0530 Subject: [PATCH 492/513] Added support for M1 build release --- release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index b26c1d44d..2d599b34c 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -581,7 +581,7 @@ stages: echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - script: | - darwin_m1_arm_dir="$(work_dir)/azcopy_darwin_m1_arm64$(azcopy_version)" + darwin_m1_arm_dir="$(work_dir)/azcopy_darwin_m1_arm64_$(azcopy_version)" echo "##vso[task.setvariable variable=darwin_m1_arm_dir]$darwin_m1_arm_dir" mkdir -p $darwin_m1_arm_dir From c37aedd6c9259272de4c6ccdb1a3410a3b63bd29 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 17:23:13 +0530 Subject: [PATCH 493/513] Added support for M1 build release --- release-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 2d599b34c..ad85428fa 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -953,7 +953,7 @@ stages: mkdir -p unzipped_m1_arm64 unzip -o azcopy*amd64*.zip -d unzipped_amd64 unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64 - unzip -o azcopy_darwin_m1_arm64_*.zip -d unzipped_m1_arm64 + unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64 # Use find to retrieve the folder name amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") @@ -962,7 +962,7 @@ stages: arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") echo "ARM64 Folder name: $arm64FolderName" - m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64_*") + m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*") echo "M1 ARM64 Folder name: $m1Arm64FolderName" # Check if there is exactly one matching directory for amd64 From ee1c67bd0495cf13db590039f4554b2b69b35e59 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 26 Jul 2024 18:11:34 +0530 Subject: [PATCH 494/513] Added support for M1 build release --- release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index ad85428fa..f06b0826b 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1813,7 +1813,7 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true if [ $(Publish_To_M1_Container) = "True" ]; then - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$(M1_CONTAINER_URL)" --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$(M1_CNT_SAS_URL)" --put-md5=true fi From 4cc789372f86bd096361a7595971bb633eebfaf8 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 11:31:55 +0530 Subject: [PATCH 495/513] incorporated review comments --- release-pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index f06b0826b..3ef47133a 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -569,7 +569,7 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" + GOOS=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" displayName: 'MacOS M1 Build with ARM64' - task: Bash@3 @@ -1713,7 +1713,6 @@ stages: vmImage: $(vmImage) variables: - - group: AZCOPY_TESTS_VAR - name: Test_Release_Account_Name value: 'azcopyprivatedrops' - name: Release_Account_Name @@ -1813,7 +1812,8 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true if [ $(Publish_To_M1_Container) = "True" ]; then - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$(M1_CNT_SAS_URL)" --put-md5=true + m1_container_url="https://$(Release_Account_Name).blob.core.windows.net/azcopy-m1-drops/azcopy-$azcopy_version-$today" + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true fi From 6ee8690b234e7331ea9081e0c4bff05954fd3b64 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 11:31:55 +0530 Subject: [PATCH 496/513] incorporated review comments --- release-pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index f06b0826b..9b1f56d56 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -569,7 +569,7 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" + GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" displayName: 'MacOS M1 Build with ARM64' - task: Bash@3 @@ -1713,7 +1713,6 @@ stages: vmImage: $(vmImage) variables: - - group: AZCOPY_TESTS_VAR - name: Test_Release_Account_Name value: 'azcopyprivatedrops' - name: Release_Account_Name @@ -1813,7 +1812,8 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true if [ $(Publish_To_M1_Container) = "True" ]; then - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$(M1_CNT_SAS_URL)" --put-md5=true + m1_container_url="https://$(Release_Account_Name).blob.core.windows.net/azcopy-m1-drops/azcopy-$azcopy_version-$today" + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true fi From 1d8e30e921f5280a6de1065449f173b093845399 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 11:45:07 +0530 Subject: [PATCH 497/513] incoporated review comments --- release-pipeline.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 56fdcce4f..3c7c19007 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -569,13 +569,9 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | -<<<<<<< HEAD GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" -======= - GOOS=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" ->>>>>>> 4cc789372f86bd096361a7595971bb633eebfaf8 displayName: 'MacOS M1 Build with ARM64' - + - task: Bash@3 displayName: 'Extract AZCopy version' inputs: From 9c8a2e152f626b7afb289f22b3b862c87da87c90 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 12:35:59 +0530 Subject: [PATCH 498/513] incoporated review comments --- release-pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 3c7c19007..949451681 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -569,9 +569,10 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory) - script: | + uname -m GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" displayName: 'MacOS M1 Build with ARM64' - + - task: Bash@3 displayName: 'Extract AZCopy version' inputs: @@ -1557,6 +1558,8 @@ stages: itemPattern: azCopy-mac-signed/azcopy_darwin_m1_arm64*.zip - script: | + uname -m + # Install Homebrew if not already installed if ! command -v brew &> /dev/null then From e0e124093cde0998caffc30fea4e102d9e140385 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 15:51:10 +0530 Subject: [PATCH 499/513] Fixed jobs list command --- ste/JobPartPlan.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ste/JobPartPlan.go b/ste/JobPartPlan.go index 852b80805..f4302d4eb 100644 --- a/ste/JobPartPlan.go +++ b/ste/JobPartPlan.go @@ -2,10 +2,11 @@ package ste import ( "errors" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "sync/atomic" "unsafe" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-azcopy/v10/common" ) @@ -142,8 +143,12 @@ func (jpph *JobPartPlanHeader) Transfer(transferIndex uint32) *JobPartPlanTransf // CommandString returns the command string given by user when job was created func (jpph *JobPartPlanHeader) CommandString() string { - data := unsafe.Pointer(uintptr(unsafe.Pointer(jpph)) + unsafe.Sizeof(*jpph)) // Address of Job Part Plan + Command String Length - return unsafe.String((*byte)(data), int(jpph.CommandStringLength)) + // Calculate the start address of the command string + start := uintptr(unsafe.Pointer(jpph)) + unsafe.Sizeof(*jpph) + + // Create a slice from the calculated start address + commandSlice := unsafe.Slice((*byte)(unsafe.Pointer(start)), int(jpph.CommandStringLength)) + return string(commandSlice) } func (jpph *JobPartPlanHeader) TransferSrcDstRelatives(transferIndex uint32) (relSource, relDest string) { From e848e3e6e164a18e23f15e26359e16ad2d4dd225 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 20:10:42 +0530 Subject: [PATCH 500/513] removed m1 build code --- release-pipeline.yml | 175 +------------------------------------------ 1 file changed, 4 insertions(+), 171 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 949451681..e4a7ceb8e 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -537,86 +537,7 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - job: Set_5 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS M1: - imageName: 'macos-14' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - uname -m - GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" - displayName: 'MacOS M1 Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_m1_arm_dir="$(work_dir)/azcopy_darwin_m1_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_m1_arm_dir]$darwin_m1_arm_dir" - - mkdir -p $darwin_m1_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_m1_arm_dir) - cp $(root_dir)/azcopy $(darwin_m1_arm_dir) - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS M1 ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_m1_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_m1_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - + #TODO: support build for mac m1 # BuildArtifacts end here - stage: SignArtifacts @@ -953,18 +874,14 @@ stages: mkdir -p unzipped_arm64 mkdir -p unzipped_m1_arm64 unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64 - unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64 + unzip -o azcopy*arm64*.zip -d unzipped_arm64 # Use find to retrieve the folder name amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") echo "AMD64 Folder name: $amd64FolderName" arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*") - echo "M1 ARM64 Folder name: $m1Arm64FolderName" + echo "ARM64 Folder name: $arm64FolderName" # Check if there is exactly one matching directory for amd64 if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then @@ -988,17 +905,6 @@ stages: exit 1 fi - cd $(Build.ArtifactStagingDirectory) - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$m1Arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching M1 ARM64 directory: $m1Arm64FolderName" - cd "$m1Arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_m1_arm64 - else - echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$m1Arm64FolderName" | wc -l)." - exit 1 - fi - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* rm -rf $(Build.ArtifactStagingDirectory)/*.zip sudo ls -ltR $(Build.ArtifactStagingDirectory) @@ -1520,80 +1426,7 @@ stages: ./azcopy --version ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - - job: Set_9 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS M1: - imageName: 'macos-14' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy_darwin_m1_arm64*.zip - - - script: | - uname -m - - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi - - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy_darwin_m1_arm64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_m1_arm64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run the azcopy version command - ./azcopy --version - ./azcopy --help - displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - + - job: Set_10 timeoutInMinutes: 120 strategy: From 8002d6b64a29abfc02c20aa79d9b6497c24f4441 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 20:12:27 +0530 Subject: [PATCH 501/513] removed m1 build code --- release-pipeline.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index e4a7ceb8e..64985cd1b 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -12,11 +12,6 @@ parameters: type: boolean default: false - - name: publish_m1_build_to_container - displayName: 'Publish M1 build To Container' - type: boolean - default: false - - name: post_release displayName: 'Post Release on Github' type: boolean @@ -1555,8 +1550,6 @@ stages: value: 'azcopyvnextrelease' - name: Publish_To_Container value: ${{ parameters.publish_to_container }} - - name: Publish_To_M1_Container - value: ${{ parameters.publish_m1_build_to_container }} - name: work_dir value: '$(Build.SourcesDirectory)' @@ -1647,12 +1640,6 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - if [ $(Publish_To_M1_Container) = "True" ]; then - m1_container_url="https://$(Release_Account_Name).blob.core.windows.net/azcopy-m1-drops/azcopy-$azcopy_version-$today" - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true - fi - - - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseArtifacts dependsOn: TestArtifacts From e8f4997fc315422c9dfb8ba827e7a15bae29aff4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 20:14:38 +0530 Subject: [PATCH 502/513] removed m1 build code --- release-pipeline.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 64985cd1b..f71185e82 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -867,7 +867,6 @@ stages: - script: | mkdir -p unzipped_amd64 mkdir -p unzipped_arm64 - mkdir -p unzipped_m1_arm64 unzip -o azcopy*amd64*.zip -d unzipped_amd64 unzip -o azcopy*arm64*.zip -d unzipped_arm64 @@ -876,7 +875,7 @@ stages: echo "AMD64 Folder name: $amd64FolderName" arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" + echo "ARM64 Folder name: $arm64FolderName" # Check if there is exactly one matching directory for amd64 if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then @@ -888,7 +887,7 @@ stages: exit 1 fi - cd $(Build.ArtifactStagingDirectory) + cd $(Build.ArtifactStagingDirectory) # Check if there is exactly one matching directory for arm64 if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then @@ -1422,7 +1421,7 @@ stages: ./azcopy --help displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - job: Set_10 + - job: Set_9 timeoutInMinutes: 120 strategy: matrix: From 1cd4ab4e0d22a9cd6791de7dc3881c1d6234bfb6 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 29 Jul 2024 15:51:10 +0530 Subject: [PATCH 503/513] Fixed jobs list command --- azure-pipelines.yml | 1 + ste/JobPartPlan.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2189cb227..6dedda333 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,6 +15,7 @@ pr: - main jobs: + - job: Build timeoutInMinutes: 360 strategy: diff --git a/ste/JobPartPlan.go b/ste/JobPartPlan.go index 852b80805..f4302d4eb 100644 --- a/ste/JobPartPlan.go +++ b/ste/JobPartPlan.go @@ -2,10 +2,11 @@ package ste import ( "errors" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "sync/atomic" "unsafe" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-azcopy/v10/common" ) @@ -142,8 +143,12 @@ func (jpph *JobPartPlanHeader) Transfer(transferIndex uint32) *JobPartPlanTransf // CommandString returns the command string given by user when job was created func (jpph *JobPartPlanHeader) CommandString() string { - data := unsafe.Pointer(uintptr(unsafe.Pointer(jpph)) + unsafe.Sizeof(*jpph)) // Address of Job Part Plan + Command String Length - return unsafe.String((*byte)(data), int(jpph.CommandStringLength)) + // Calculate the start address of the command string + start := uintptr(unsafe.Pointer(jpph)) + unsafe.Sizeof(*jpph) + + // Create a slice from the calculated start address + commandSlice := unsafe.Slice((*byte)(unsafe.Pointer(start)), int(jpph.CommandStringLength)) + return string(commandSlice) } func (jpph *JobPartPlanHeader) TransferSrcDstRelatives(transferIndex uint32) (relSource, relDest string) { From 3aab3ee681493824ec8ee4dbc2f8ded56339b685 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 30 Jul 2024 11:30:35 +0530 Subject: [PATCH 504/513] incorporated review comments --- release-pipeline.yml | 63 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index f71185e82..a5218d0f7 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1888,35 +1888,34 @@ stages: steps: - checkout: none - - ${{ if eq(parameters.update_version, true) }}: - - script: | - sudo apt-get install python3 -y - python3 --version - displayName: 'Installing Python' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - # install azcopy - - script: | - cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - ls azcopy*.arm64.deb - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - azcopy --version - displayName: 'Installing azcopy' - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py - ls -l - displayName: 'Getting Python script' - workingDirectory: $(root_dir) - - - script: | - pip install azure-storage-blob - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" - displayName: 'Updating version number' - workingDirectory: $(root_dir) + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + # install azcopy + - script: | + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir) From a75c575692a13e83f477156e88fe83523af05580 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 14 Aug 2024 16:16:05 +0000 Subject: [PATCH 505/513] Add testcase --- e2etest/newe2e_jobs_list_test.go | 19 +++++++++++++++++++ e2etest/newe2e_task_runazcopy.go | 18 ++++++++++-------- e2etest/zt_aanewe2e_testmain_test.go | 16 ++++++++++++++++ .../zt_newe2e_autodetect_blob_type_test.go | 2 +- e2etest/zt_newe2e_basic_functionality_test.go | 5 +++-- e2etest/zt_newe2e_blob_test.go | 5 +++-- e2etest/zt_newe2e_blobfs_test.go | 5 +++-- e2etest/zt_newe2e_file_oauth_test.go | 3 ++- e2etest/zt_newe2e_file_test.go | 7 ++++--- e2etest/zt_newe2e_list_test.go | 2 +- e2etest/zt_newe2e_remove_test.go | 5 +++-- e2etest/zt_newe2e_s2s_test.go | 5 +++-- e2etest/zt_newe2e_sync_test.go | 5 +++-- e2etest/zt_newe2e_workload_test.go | 5 +++-- 14 files changed, 74 insertions(+), 28 deletions(-) create mode 100644 e2etest/newe2e_jobs_list_test.go diff --git a/e2etest/newe2e_jobs_list_test.go b/e2etest/newe2e_jobs_list_test.go new file mode 100644 index 000000000..46d8d1698 --- /dev/null +++ b/e2etest/newe2e_jobs_list_test.go @@ -0,0 +1,19 @@ +package e2etest + +import "fmt" + +func init() { + suiteManager.RegisterSuite(&JobsListSuite{}) +} + +type JobsListSuite struct{} + +func (s *JobsListSuite) Scenario_BasicJobsList(svm *ScenarioVariationManager) { + stdout, _ := RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbJobsList, + }) + + fmt.Println("STDOUT*********", stdout) +} diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index 1145c8aeb..b867b1291 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -2,7 +2,6 @@ package e2etest import ( "fmt" - "github.com/Azure/azure-storage-azcopy/v10/common" "io" "io/fs" "os" @@ -10,6 +9,8 @@ import ( "path/filepath" "reflect" "strings" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) // AzCopyJobPlan todo probably load the job plan directly? WI#26418256 @@ -49,10 +50,11 @@ var _ AzCopyStdout = &AzCopyRawStdout{} type AzCopyVerb string const ( // initially supporting a limited set of verbs - AzCopyVerbCopy AzCopyVerb = "copy" - AzCopyVerbSync AzCopyVerb = "sync" - AzCopyVerbRemove AzCopyVerb = "remove" - AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbCopy AzCopyVerb = "copy" + AzCopyVerbSync AzCopyVerb = "sync" + AzCopyVerbRemove AzCopyVerb = "remove" + AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbJobsList AzCopyVerb = "jobs list" ) type AzCopyTarget struct { @@ -236,9 +238,9 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri // RunAzCopy todo define more cleanly, implement func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *AzCopyJobPlan) { - if a.Dryrun() { - return nil, &AzCopyJobPlan{} - } + // if a.Dryrun() { + // return nil, &AzCopyJobPlan{} + // } var flagMap map[string]string var envMap map[string]string diff --git a/e2etest/zt_aanewe2e_testmain_test.go b/e2etest/zt_aanewe2e_testmain_test.go index a0ea73f69..8885c4ea0 100644 --- a/e2etest/zt_aanewe2e_testmain_test.go +++ b/e2etest/zt_aanewe2e_testmain_test.go @@ -1,6 +1,7 @@ package e2etest import ( + "os" "testing" ) @@ -31,6 +32,21 @@ type TestFrameworkHook struct { func TestNewE2E(t *testing.T) { a := &FrameworkAsserter{t: t} + os.Setenv("NEW_E2E_APPLICATION_ID", "6e664d29-7531-4a2f-8ec8-f04ac1f99b6c") + os.Setenv("NEW_E2E_TENANT_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") + os.Setenv("NEW_E2E_CLIENT_SECRET", "OMx8Q~1fP~X-UTgKGBn_j3vNBP~tRYaarYqwwa-.") + os.Setenv("NEW_E2E_STATIC_APPLICATION_ID", "6e664d29-7531-4a2f-8ec8-f04ac1f99b6c") + os.Setenv("NEW_E2E_STATIC_TENANT_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") + os.Setenv("NEW_E2E_APPLICATION_ID", "6e664d29-7531-4a2f-8ec8-f04ac1f99b6c") + os.Setenv("NEW_E2E_STATIC_CLIENT_SECRET", "OMx8Q~1fP~X-UTgKGBn_j3vNBP~tRYaarYqwwa-.") + os.Setenv("AZCOPY_E2E_LOG_OUTPUT", "/home/azureuser") + os.Setenv("NEW_E2E_AZCOPY_PATH", "/home/azureuser/go/src/azure-storage-azcopy/azcopy") + os.Setenv("NEW_E2E_STANDARD_ACCOUNT_NAME", "azcopye2epipeline") + os.Setenv("NEW_E2E_STANDARD_ACCOUNT_KEY", "U6IPcw4JXZQFmLAQ1GgBWLIcW6phz3VdJGwR1amXhSFkP8FqzKxFMaqnkW3K8/JamTllYGGYtknXVOT8jCKVAg==") + os.Setenv("NEW_E2E_HNS_ACCOUNT_NAME", "azcopye2ehnstest1") + os.Setenv("NEW_E2E_HNS_ACCOUNT_NAME", "azcopye2ehnstest1") + os.Setenv("NEW_E2E_SUBSCRIPTION_ID", "") + t.Cleanup(func() { for i := len(FrameworkHooks) - 1; i >= 0; i-- { hook := FrameworkHooks[i] diff --git a/e2etest/zt_newe2e_autodetect_blob_type_test.go b/e2etest/zt_newe2e_autodetect_blob_type_test.go index 1762209e1..2ea97a918 100644 --- a/e2etest/zt_newe2e_autodetect_blob_type_test.go +++ b/e2etest/zt_newe2e_autodetect_blob_type_test.go @@ -6,7 +6,7 @@ import ( ) func init() { - suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{}) + //suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{}) } type AutoDetectBlobTypeTestSuite struct{} diff --git a/e2etest/zt_newe2e_basic_functionality_test.go b/e2etest/zt_newe2e_basic_functionality_test.go index ebf7f0380..dfc7c9f8b 100644 --- a/e2etest/zt_newe2e_basic_functionality_test.go +++ b/e2etest/zt_newe2e_basic_functionality_test.go @@ -1,13 +1,14 @@ package e2etest import ( - "github.com/Azure/azure-storage-azcopy/v10/common" "strconv" "time" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - suiteManager.RegisterSuite(&BasicFunctionalitySuite{}) + //suiteManager.RegisterSuite(&BasicFunctionalitySuite{}) } type BasicFunctionalitySuite struct{} diff --git a/e2etest/zt_newe2e_blob_test.go b/e2etest/zt_newe2e_blob_test.go index b1f4b6b60..c50b9d9c3 100644 --- a/e2etest/zt_newe2e_blob_test.go +++ b/e2etest/zt_newe2e_blob_test.go @@ -1,13 +1,14 @@ package e2etest import ( + "strconv" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-storage-azcopy/v10/common" - "strconv" ) func init() { - suiteManager.RegisterSuite(&BlobTestSuite{}) + //suiteManager.RegisterSuite(&BlobTestSuite{}) } type BlobTestSuite struct{} diff --git a/e2etest/zt_newe2e_blobfs_test.go b/e2etest/zt_newe2e_blobfs_test.go index dfba6d850..c3085f354 100644 --- a/e2etest/zt_newe2e_blobfs_test.go +++ b/e2etest/zt_newe2e_blobfs_test.go @@ -1,12 +1,13 @@ package e2etest import ( - "github.com/Azure/azure-storage-azcopy/v10/common" "strconv" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - suiteManager.RegisterSuite(&BlobFSTestSuite{}) + //suiteManager.RegisterSuite(&BlobFSTestSuite{}) } type BlobFSTestSuite struct{} diff --git a/e2etest/zt_newe2e_file_oauth_test.go b/e2etest/zt_newe2e_file_oauth_test.go index 95c1de0c7..4e89d71aa 100644 --- a/e2etest/zt_newe2e_file_oauth_test.go +++ b/e2etest/zt_newe2e_file_oauth_test.go @@ -2,11 +2,12 @@ package e2etest import ( "fmt" + "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - suiteManager.RegisterSuite(&FileOAuthTestSuite{}) + //suiteManager.RegisterSuite(&FileOAuthTestSuite{}) } type FileOAuthTestSuite struct{} diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index 48d425297..eede532df 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -3,13 +3,14 @@ package e2etest import ( "context" "fmt" - "github.com/Azure/azure-storage-azcopy/v10/common" - "strconv" "math" + "strconv" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - suiteManager.RegisterSuite(&FileTestSuite{}) + //suiteManager.RegisterSuite(&FileTestSuite{}) } type FileTestSuite struct{} diff --git a/e2etest/zt_newe2e_list_test.go b/e2etest/zt_newe2e_list_test.go index c58e1302f..17bd132b6 100644 --- a/e2etest/zt_newe2e_list_test.go +++ b/e2etest/zt_newe2e_list_test.go @@ -8,7 +8,7 @@ import ( ) func init() { - suiteManager.RegisterSuite(&ListSuite{}) + //suiteManager.RegisterSuite(&ListSuite{}) } type ListSuite struct{} diff --git a/e2etest/zt_newe2e_remove_test.go b/e2etest/zt_newe2e_remove_test.go index 3d5408a3e..c11a65e54 100644 --- a/e2etest/zt_newe2e_remove_test.go +++ b/e2etest/zt_newe2e_remove_test.go @@ -1,14 +1,15 @@ package e2etest import ( + "strconv" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-storage-azcopy/v10/common" - "strconv" ) func init() { - suiteManager.RegisterSuite(&RemoveSuite{}) + //suiteManager.RegisterSuite(&RemoveSuite{}) } type RemoveSuite struct{} diff --git a/e2etest/zt_newe2e_s2s_test.go b/e2etest/zt_newe2e_s2s_test.go index b286196b4..3ab0d4c98 100644 --- a/e2etest/zt_newe2e_s2s_test.go +++ b/e2etest/zt_newe2e_s2s_test.go @@ -1,13 +1,14 @@ package e2etest import ( + "strconv" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-storage-azcopy/v10/common" - "strconv" ) func init() { - suiteManager.RegisterSuite(&S2STestSuite{}) + //suiteManager.RegisterSuite(&S2STestSuite{}) } type S2STestSuite struct{} diff --git a/e2etest/zt_newe2e_sync_test.go b/e2etest/zt_newe2e_sync_test.go index 66529de10..08ec9e566 100644 --- a/e2etest/zt_newe2e_sync_test.go +++ b/e2etest/zt_newe2e_sync_test.go @@ -2,18 +2,19 @@ package e2etest import ( "encoding/base64" - "github.com/Azure/azure-storage-azcopy/v10/common" "io/fs" "os" "path/filepath" "runtime" "time" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) type SyncTestSuite struct{} func init() { - suiteManager.RegisterSuite(&SyncTestSuite{}) + //suiteManager.RegisterSuite(&SyncTestSuite{}) } func (s *SyncTestSuite) Scenario_TestSyncHashStorageModes(a *ScenarioVariationManager) { diff --git a/e2etest/zt_newe2e_workload_test.go b/e2etest/zt_newe2e_workload_test.go index faf2ff496..519765dd1 100644 --- a/e2etest/zt_newe2e_workload_test.go +++ b/e2etest/zt_newe2e_workload_test.go @@ -1,13 +1,14 @@ package e2etest import ( - "github.com/Azure/azure-storage-azcopy/v10/common" "os" "time" + + "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - suiteManager.RegisterSuite(&WorkloadIdentitySuite{}) + //suiteManager.RegisterSuite(&WorkloadIdentitySuite{}) } type WorkloadIdentitySuite struct{} From 1e7a3c016f222c680dcc88465b3584faba64e6b9 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 20 Aug 2024 15:45:24 +0530 Subject: [PATCH 506/513] code changes --- packages.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.csv b/packages.csv index 21d9ae9f3..0c315d064 100644 --- a/packages.csv +++ b/packages.csv @@ -25,4 +25,4 @@ SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, Mariner-3.0-x86_64,marinerAmdRpm,azurelinux-3.0-prod-ms-oss-x86_64-yum, -Mariner-3.0-aarch64,marinerAarchRpm,azurelinux-3.0-prod-ms-oss-aarch64-yum, +Mariner-3.0-aarch64,marinerAarchRpm,azurelinux-3.0-prod-ms-oss-aarch64-yum, \ No newline at end of file From 19b538752974159279a9c1a2c83b96d15ec2b083 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 21 Aug 2024 10:46:44 +0530 Subject: [PATCH 507/513] revert unnecessary changes --- e2etest/newe2e_jobs_list_test.go | 19 ------------------- e2etest/newe2e_task_runazcopy.go | 18 ++++++++---------- e2etest/zt_aanewe2e_testmain_test.go | 16 ---------------- .../zt_newe2e_autodetect_blob_type_test.go | 2 +- e2etest/zt_newe2e_basic_functionality_test.go | 5 ++--- e2etest/zt_newe2e_blob_test.go | 5 ++--- e2etest/zt_newe2e_blobfs_test.go | 5 ++--- e2etest/zt_newe2e_file_oauth_test.go | 3 +-- e2etest/zt_newe2e_file_test.go | 7 +++---- e2etest/zt_newe2e_list_test.go | 2 +- e2etest/zt_newe2e_remove_test.go | 5 ++--- e2etest/zt_newe2e_s2s_test.go | 5 ++--- e2etest/zt_newe2e_sync_test.go | 5 ++--- e2etest/zt_newe2e_workload_test.go | 5 ++--- 14 files changed, 28 insertions(+), 74 deletions(-) delete mode 100644 e2etest/newe2e_jobs_list_test.go diff --git a/e2etest/newe2e_jobs_list_test.go b/e2etest/newe2e_jobs_list_test.go deleted file mode 100644 index 46d8d1698..000000000 --- a/e2etest/newe2e_jobs_list_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package e2etest - -import "fmt" - -func init() { - suiteManager.RegisterSuite(&JobsListSuite{}) -} - -type JobsListSuite struct{} - -func (s *JobsListSuite) Scenario_BasicJobsList(svm *ScenarioVariationManager) { - stdout, _ := RunAzCopy( - svm, - AzCopyCommand{ - Verb: AzCopyVerbJobsList, - }) - - fmt.Println("STDOUT*********", stdout) -} diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index b867b1291..1145c8aeb 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -2,6 +2,7 @@ package e2etest import ( "fmt" + "github.com/Azure/azure-storage-azcopy/v10/common" "io" "io/fs" "os" @@ -9,8 +10,6 @@ import ( "path/filepath" "reflect" "strings" - - "github.com/Azure/azure-storage-azcopy/v10/common" ) // AzCopyJobPlan todo probably load the job plan directly? WI#26418256 @@ -50,11 +49,10 @@ var _ AzCopyStdout = &AzCopyRawStdout{} type AzCopyVerb string const ( // initially supporting a limited set of verbs - AzCopyVerbCopy AzCopyVerb = "copy" - AzCopyVerbSync AzCopyVerb = "sync" - AzCopyVerbRemove AzCopyVerb = "remove" - AzCopyVerbList AzCopyVerb = "list" - AzCopyVerbJobsList AzCopyVerb = "jobs list" + AzCopyVerbCopy AzCopyVerb = "copy" + AzCopyVerbSync AzCopyVerb = "sync" + AzCopyVerbRemove AzCopyVerb = "remove" + AzCopyVerbList AzCopyVerb = "list" ) type AzCopyTarget struct { @@ -238,9 +236,9 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri // RunAzCopy todo define more cleanly, implement func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *AzCopyJobPlan) { - // if a.Dryrun() { - // return nil, &AzCopyJobPlan{} - // } + if a.Dryrun() { + return nil, &AzCopyJobPlan{} + } var flagMap map[string]string var envMap map[string]string diff --git a/e2etest/zt_aanewe2e_testmain_test.go b/e2etest/zt_aanewe2e_testmain_test.go index 8885c4ea0..a0ea73f69 100644 --- a/e2etest/zt_aanewe2e_testmain_test.go +++ b/e2etest/zt_aanewe2e_testmain_test.go @@ -1,7 +1,6 @@ package e2etest import ( - "os" "testing" ) @@ -32,21 +31,6 @@ type TestFrameworkHook struct { func TestNewE2E(t *testing.T) { a := &FrameworkAsserter{t: t} - os.Setenv("NEW_E2E_APPLICATION_ID", "6e664d29-7531-4a2f-8ec8-f04ac1f99b6c") - os.Setenv("NEW_E2E_TENANT_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") - os.Setenv("NEW_E2E_CLIENT_SECRET", "OMx8Q~1fP~X-UTgKGBn_j3vNBP~tRYaarYqwwa-.") - os.Setenv("NEW_E2E_STATIC_APPLICATION_ID", "6e664d29-7531-4a2f-8ec8-f04ac1f99b6c") - os.Setenv("NEW_E2E_STATIC_TENANT_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") - os.Setenv("NEW_E2E_APPLICATION_ID", "6e664d29-7531-4a2f-8ec8-f04ac1f99b6c") - os.Setenv("NEW_E2E_STATIC_CLIENT_SECRET", "OMx8Q~1fP~X-UTgKGBn_j3vNBP~tRYaarYqwwa-.") - os.Setenv("AZCOPY_E2E_LOG_OUTPUT", "/home/azureuser") - os.Setenv("NEW_E2E_AZCOPY_PATH", "/home/azureuser/go/src/azure-storage-azcopy/azcopy") - os.Setenv("NEW_E2E_STANDARD_ACCOUNT_NAME", "azcopye2epipeline") - os.Setenv("NEW_E2E_STANDARD_ACCOUNT_KEY", "U6IPcw4JXZQFmLAQ1GgBWLIcW6phz3VdJGwR1amXhSFkP8FqzKxFMaqnkW3K8/JamTllYGGYtknXVOT8jCKVAg==") - os.Setenv("NEW_E2E_HNS_ACCOUNT_NAME", "azcopye2ehnstest1") - os.Setenv("NEW_E2E_HNS_ACCOUNT_NAME", "azcopye2ehnstest1") - os.Setenv("NEW_E2E_SUBSCRIPTION_ID", "") - t.Cleanup(func() { for i := len(FrameworkHooks) - 1; i >= 0; i-- { hook := FrameworkHooks[i] diff --git a/e2etest/zt_newe2e_autodetect_blob_type_test.go b/e2etest/zt_newe2e_autodetect_blob_type_test.go index 2ea97a918..1762209e1 100644 --- a/e2etest/zt_newe2e_autodetect_blob_type_test.go +++ b/e2etest/zt_newe2e_autodetect_blob_type_test.go @@ -6,7 +6,7 @@ import ( ) func init() { - //suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{}) + suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{}) } type AutoDetectBlobTypeTestSuite struct{} diff --git a/e2etest/zt_newe2e_basic_functionality_test.go b/e2etest/zt_newe2e_basic_functionality_test.go index dfc7c9f8b..ebf7f0380 100644 --- a/e2etest/zt_newe2e_basic_functionality_test.go +++ b/e2etest/zt_newe2e_basic_functionality_test.go @@ -1,14 +1,13 @@ package e2etest import ( + "github.com/Azure/azure-storage-azcopy/v10/common" "strconv" "time" - - "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - //suiteManager.RegisterSuite(&BasicFunctionalitySuite{}) + suiteManager.RegisterSuite(&BasicFunctionalitySuite{}) } type BasicFunctionalitySuite struct{} diff --git a/e2etest/zt_newe2e_blob_test.go b/e2etest/zt_newe2e_blob_test.go index c50b9d9c3..b1f4b6b60 100644 --- a/e2etest/zt_newe2e_blob_test.go +++ b/e2etest/zt_newe2e_blob_test.go @@ -1,14 +1,13 @@ package e2etest import ( - "strconv" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" ) func init() { - //suiteManager.RegisterSuite(&BlobTestSuite{}) + suiteManager.RegisterSuite(&BlobTestSuite{}) } type BlobTestSuite struct{} diff --git a/e2etest/zt_newe2e_blobfs_test.go b/e2etest/zt_newe2e_blobfs_test.go index c3085f354..dfba6d850 100644 --- a/e2etest/zt_newe2e_blobfs_test.go +++ b/e2etest/zt_newe2e_blobfs_test.go @@ -1,13 +1,12 @@ package e2etest import ( - "strconv" - "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" ) func init() { - //suiteManager.RegisterSuite(&BlobFSTestSuite{}) + suiteManager.RegisterSuite(&BlobFSTestSuite{}) } type BlobFSTestSuite struct{} diff --git a/e2etest/zt_newe2e_file_oauth_test.go b/e2etest/zt_newe2e_file_oauth_test.go index 4e89d71aa..95c1de0c7 100644 --- a/e2etest/zt_newe2e_file_oauth_test.go +++ b/e2etest/zt_newe2e_file_oauth_test.go @@ -2,12 +2,11 @@ package e2etest import ( "fmt" - "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - //suiteManager.RegisterSuite(&FileOAuthTestSuite{}) + suiteManager.RegisterSuite(&FileOAuthTestSuite{}) } type FileOAuthTestSuite struct{} diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index eede532df..48d425297 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -3,14 +3,13 @@ package e2etest import ( "context" "fmt" - "math" - "strconv" - "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" + "math" ) func init() { - //suiteManager.RegisterSuite(&FileTestSuite{}) + suiteManager.RegisterSuite(&FileTestSuite{}) } type FileTestSuite struct{} diff --git a/e2etest/zt_newe2e_list_test.go b/e2etest/zt_newe2e_list_test.go index 17bd132b6..c58e1302f 100644 --- a/e2etest/zt_newe2e_list_test.go +++ b/e2etest/zt_newe2e_list_test.go @@ -8,7 +8,7 @@ import ( ) func init() { - //suiteManager.RegisterSuite(&ListSuite{}) + suiteManager.RegisterSuite(&ListSuite{}) } type ListSuite struct{} diff --git a/e2etest/zt_newe2e_remove_test.go b/e2etest/zt_newe2e_remove_test.go index c11a65e54..3d5408a3e 100644 --- a/e2etest/zt_newe2e_remove_test.go +++ b/e2etest/zt_newe2e_remove_test.go @@ -1,15 +1,14 @@ package e2etest import ( - "strconv" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" ) func init() { - //suiteManager.RegisterSuite(&RemoveSuite{}) + suiteManager.RegisterSuite(&RemoveSuite{}) } type RemoveSuite struct{} diff --git a/e2etest/zt_newe2e_s2s_test.go b/e2etest/zt_newe2e_s2s_test.go index 3ab0d4c98..b286196b4 100644 --- a/e2etest/zt_newe2e_s2s_test.go +++ b/e2etest/zt_newe2e_s2s_test.go @@ -1,14 +1,13 @@ package e2etest import ( - "strconv" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" ) func init() { - //suiteManager.RegisterSuite(&S2STestSuite{}) + suiteManager.RegisterSuite(&S2STestSuite{}) } type S2STestSuite struct{} diff --git a/e2etest/zt_newe2e_sync_test.go b/e2etest/zt_newe2e_sync_test.go index 08ec9e566..66529de10 100644 --- a/e2etest/zt_newe2e_sync_test.go +++ b/e2etest/zt_newe2e_sync_test.go @@ -2,19 +2,18 @@ package e2etest import ( "encoding/base64" + "github.com/Azure/azure-storage-azcopy/v10/common" "io/fs" "os" "path/filepath" "runtime" "time" - - "github.com/Azure/azure-storage-azcopy/v10/common" ) type SyncTestSuite struct{} func init() { - //suiteManager.RegisterSuite(&SyncTestSuite{}) + suiteManager.RegisterSuite(&SyncTestSuite{}) } func (s *SyncTestSuite) Scenario_TestSyncHashStorageModes(a *ScenarioVariationManager) { diff --git a/e2etest/zt_newe2e_workload_test.go b/e2etest/zt_newe2e_workload_test.go index 519765dd1..faf2ff496 100644 --- a/e2etest/zt_newe2e_workload_test.go +++ b/e2etest/zt_newe2e_workload_test.go @@ -1,14 +1,13 @@ package e2etest import ( + "github.com/Azure/azure-storage-azcopy/v10/common" "os" "time" - - "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { - //suiteManager.RegisterSuite(&WorkloadIdentitySuite{}) + suiteManager.RegisterSuite(&WorkloadIdentitySuite{}) } type WorkloadIdentitySuite struct{} From 6552e256b74b5cd474b4b53d76b5743b248535d4 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 21 Aug 2024 10:52:21 +0530 Subject: [PATCH 508/513] revert unnecessary changes --- packages.csv | 2 +- setup/test_artifacts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.csv b/packages.csv index 0c315d064..21d9ae9f3 100644 --- a/packages.csv +++ b/packages.csv @@ -25,4 +25,4 @@ SUSE-15Gen2,azcopyAmdRpm,microsoft-sles15-prod-yum, Rocky-8.0,azcopyAmdRpm,microsoft-el8-prod-yum, Rocky-9.0,azcopyAmdRpm,microsoft-el9-prod-yum, Mariner-3.0-x86_64,marinerAmdRpm,azurelinux-3.0-prod-ms-oss-x86_64-yum, -Mariner-3.0-aarch64,marinerAarchRpm,azurelinux-3.0-prod-ms-oss-aarch64-yum, \ No newline at end of file +Mariner-3.0-aarch64,marinerAarchRpm,azurelinux-3.0-prod-ms-oss-aarch64-yum, diff --git a/setup/test_artifacts.yml b/setup/test_artifacts.yml index 098ea23e7..790a729b5 100644 --- a/setup/test_artifacts.yml +++ b/setup/test_artifacts.yml @@ -35,4 +35,4 @@ steps: - script: | ls -l displayName: 'List Downloaded Package' - workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} \ No newline at end of file + workingDirectory: ${{ parameters.download_path }}/${{ parameters.artifact_name }} From 0182beb710c81b10ef2361270282118587a40324 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Fri, 30 Aug 2024 12:08:07 +0530 Subject: [PATCH 509/513] Added test case --- e2etest/newe2e_runazcopy_stdout.go | 24 ++++++++++++++++++++++++ e2etest/newe2e_task_runazcopy.go | 18 +++++++++++++----- e2etest/newe2e_task_validation.go | 10 ++++++++++ e2etest/zt_newe2e_jobs_list_test.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 e2etest/zt_newe2e_jobs_list_test.go diff --git a/e2etest/newe2e_runazcopy_stdout.go b/e2etest/newe2e_runazcopy_stdout.go index 9c64e7069..9be719f28 100644 --- a/e2etest/newe2e_runazcopy_stdout.go +++ b/e2etest/newe2e_runazcopy_stdout.go @@ -11,6 +11,7 @@ var _ AzCopyStdout = &AzCopyParsedStdout{} var _ AzCopyStdout = &AzCopyParsedListStdout{} var _ AzCopyStdout = &AzCopyParsedCopySyncRemoveStdout{} var _ AzCopyStdout = &AzCopyParsedDryrunStdout{} +var _ AzCopyStdout = &AzCopyParsedJobsListStdout{} // ManySubscriberChannel is intended to reproduce the effects of .NET's events. // This allows us to *partially* answer the question of how we want to handle testing of prompting in the New E2E framework. @@ -194,3 +195,26 @@ func (a *AzCopyParsedDryrunStdout) Write(p []byte) (n int, err error) { return a.AzCopyParsedStdout.Write(p) } + +type AzCopyParsedJobsListStdout struct { + AzCopyParsedStdout + listenChan chan<- common.JsonOutputTemplate + JobsCount int +} + +func (a *AzCopyParsedJobsListStdout) Write(p []byte) (n int, err error) { + if a.listenChan == nil { + a.listenChan = a.OnParsedLine.SubscribeFunc(func(line common.JsonOutputTemplate) { + if line.MessageType == common.EOutputMessageType.EndOfJob().String() { + var tx common.ListJobsResponse + err = json.Unmarshal([]byte(line.MessageContent), &tx) + if err != nil { + return + } + + a.JobsCount = len(tx.JobIDDetails) + } + }) + } + return a.AzCopyParsedStdout.Write(p) +} diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index 1145c8aeb..3709be4bb 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -49,10 +49,11 @@ var _ AzCopyStdout = &AzCopyRawStdout{} type AzCopyVerb string const ( // initially supporting a limited set of verbs - AzCopyVerbCopy AzCopyVerb = "copy" - AzCopyVerbSync AzCopyVerb = "sync" - AzCopyVerbRemove AzCopyVerb = "remove" - AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbCopy AzCopyVerb = "copy" + AzCopyVerbSync AzCopyVerb = "sync" + AzCopyVerbRemove AzCopyVerb = "remove" + AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbJobsList AzCopyVerb = "jobs" ) type AzCopyTarget struct { @@ -89,7 +90,8 @@ func CreateAzCopyTarget(rm ResourceManager, authType ExplicitCredentialTypes, a } type AzCopyCommand struct { - Verb AzCopyVerb + Verb AzCopyVerb + PositionalArgs []string // Passing a ResourceManager assumes SAS (or GCP/S3) auth is intended. // Passing an AzCopyTarget will allow you to specify an exact credential type. // When OAuth, S3, GCP, AcctKey, etc. the appropriate env flags should auto-populate. @@ -249,6 +251,10 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az } out := []string{GlobalConfig.AzCopyExecutableConfig.ExecutablePath, string(commandSpec.Verb)} + + for _, v := range commandSpec.PositionalArgs { + out = append(out, v) + } for _, v := range commandSpec.Targets { out = append(out, commandSpec.applyTargetAuth(a, v)) @@ -290,6 +296,8 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az out = &AzCopyParsedCopySyncRemoveStdout{} case commandSpec.Verb == AzCopyVerbList: out = &AzCopyParsedListStdout{} + case commandSpec.Verb == AzCopyVerbJobsList: + out = &AzCopyParsedJobsListStdout{} default: // We don't know how to parse this. out = &AzCopyRawStdout{} } diff --git a/e2etest/newe2e_task_validation.go b/e2etest/newe2e_task_validation.go index 22b44943c..3b1d49f0c 100644 --- a/e2etest/newe2e_task_validation.go +++ b/e2etest/newe2e_task_validation.go @@ -308,3 +308,13 @@ func parseAzCopyListObject(a Asserter, line string) cmd.AzCopyListObject { ContentLength: properties["Content Length"], } } + +func ValidateJobsListOutput(a Asserter, stdout AzCopyStdout, expectedJobIDs int) { + if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() { + return + } + + jobsListStdout, ok := stdout.(*AzCopyParsedJobsListStdout) + a.AssertNow("stdout must be AzCopyParsedJobsListStdout", Equal{}, ok, true) + a.Assert("map of objects must be equivalent in size", Equal{}, expectedJobIDs, jobsListStdout.JobsCount) +} diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go new file mode 100644 index 000000000..5bae6a63e --- /dev/null +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -0,0 +1,29 @@ +package e2etest + +import ( + "fmt" +) + +func init() { + suiteManager.RegisterSuite(&JobsListSuite{}) +} + +type JobsListSuite struct{} + +func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { + //_, _ = RunAzCopy( + // svm, + // AzCopyCommand{ + // Verb: AzCopyVerbJobsList, + // PositionalArgs: []string{"list"}, + // }) + + jloutput, _ := RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbJobsList, + PositionalArgs: []string{"list"}, + }) + ValidateJobsListOutput(svm, jloutput, 0) + fmt.Println("stdout Output: ", jloutput) +} From 03d8001c7157ba8ea5788f404b5a19f15c47f3b3 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 2 Sep 2024 10:29:47 +0530 Subject: [PATCH 510/513] Added test case --- e2etest/zt_newe2e_jobs_list_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index 5bae6a63e..267cda30f 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -23,6 +23,7 @@ func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { AzCopyCommand{ Verb: AzCopyVerbJobsList, PositionalArgs: []string{"list"}, + Stdout: &AzCopyParsedJobsListStdout{}, }) ValidateJobsListOutput(svm, jloutput, 0) fmt.Println("stdout Output: ", jloutput) From d26f60d0ee65c575f36689887ae546c3f50f337c Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Mon, 2 Sep 2024 11:54:06 +0530 Subject: [PATCH 511/513] Added test case --- e2etest/newe2e_task_validation.go | 11 +++--- e2etest/zt_newe2e_jobs_list_test.go | 52 +++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/e2etest/newe2e_task_validation.go b/e2etest/newe2e_task_validation.go index 3b1d49f0c..4d0c6fed3 100644 --- a/e2etest/newe2e_task_validation.go +++ b/e2etest/newe2e_task_validation.go @@ -5,14 +5,15 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" - "github.com/Azure/azure-storage-azcopy/v10/cmd" - "github.com/Azure/azure-storage-azcopy/v10/common" "io" "reflect" "strings" "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" + "github.com/Azure/azure-storage-azcopy/v10/cmd" + "github.com/Azure/azure-storage-azcopy/v10/common" ) func ValidatePropertyPtr[T any](a Asserter, name string, expected, real *T) { @@ -316,5 +317,5 @@ func ValidateJobsListOutput(a Asserter, stdout AzCopyStdout, expectedJobIDs int) jobsListStdout, ok := stdout.(*AzCopyParsedJobsListStdout) a.AssertNow("stdout must be AzCopyParsedJobsListStdout", Equal{}, ok, true) - a.Assert("map of objects must be equivalent in size", Equal{}, expectedJobIDs, jobsListStdout.JobsCount) + a.Assert("No of jobs executed should be equivalent", Equal{}, expectedJobIDs, jobsListStdout.JobsCount) } diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index 267cda30f..11dfd30f0 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -2,6 +2,9 @@ package e2etest import ( "fmt" + blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/Azure/azure-storage-azcopy/v10/cmd" + "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { @@ -11,20 +14,49 @@ func init() { type JobsListSuite struct{} func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { - //_, _ = RunAzCopy( - // svm, - // AzCopyCommand{ - // Verb: AzCopyVerbJobsList, - // PositionalArgs: []string{"list"}, - // }) - - jloutput, _ := RunAzCopy( + srcService := GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob()})) + + svm.InsertVariationSeparator(":") + body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) + var expectedObjects map[AzCopyOutputKey]cmd.AzCopyListObject + if srcService.Location() == common.ELocation.Blob() { + expectedObjects = map[AzCopyOutputKey]cmd.AzCopyListObject{} + } else { + expectedObjects = map[AzCopyOutputKey]cmd.AzCopyListObject{ + AzCopyOutputKey{Path: "/"}: {Path: "/", ContentLength: "0.00 B"}, + } + } + // Scale up from service to object + srcObj := CreateResource[ObjectResourceManager](svm, srcService, ResourceDefinitionObject{ + ObjectName: pointerTo("test"), + Body: body, + }) + expectedObjects[AzCopyOutputKey{Path: "test"}] = cmd.AzCopyListObject{Path: "test", ContentLength: "1.00 KiB"} + + stdout, _ := RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbList, + Targets: []ResourceManager{ + srcObj.Parent().(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: GenericServiceSignatureValues{ + ContainerName: srcObj.ContainerName(), + Permissions: (&blobsas.ContainerPermissions{Read: true, List: true}).String(), + }, + }), + }, + Flags: ListFlags{}, + }) + + ValidateListOutput(svm, stdout, expectedObjects, nil) + + jobsListOutput, _ := RunAzCopy( svm, AzCopyCommand{ Verb: AzCopyVerbJobsList, PositionalArgs: []string{"list"}, Stdout: &AzCopyParsedJobsListStdout{}, }) - ValidateJobsListOutput(svm, jloutput, 0) - fmt.Println("stdout Output: ", jloutput) + ValidateJobsListOutput(svm, jobsListOutput, 1) + fmt.Println("stdout Output: ", jobsListOutput) } From 7a08400d55b212d2ba0b48a5d8d7b52213105b37 Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Tue, 17 Sep 2024 21:43:32 +0530 Subject: [PATCH 512/513] Add test case --- e2etest/zt_newe2e_jobs_list_test.go | 41 ++--------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index 11dfd30f0..135ee3c59 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -2,9 +2,6 @@ package e2etest import ( "fmt" - blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" - "github.com/Azure/azure-storage-azcopy/v10/cmd" - "github.com/Azure/azure-storage-azcopy/v10/common" ) func init() { @@ -14,41 +11,6 @@ func init() { type JobsListSuite struct{} func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { - srcService := GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob()})) - - svm.InsertVariationSeparator(":") - body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) - var expectedObjects map[AzCopyOutputKey]cmd.AzCopyListObject - if srcService.Location() == common.ELocation.Blob() { - expectedObjects = map[AzCopyOutputKey]cmd.AzCopyListObject{} - } else { - expectedObjects = map[AzCopyOutputKey]cmd.AzCopyListObject{ - AzCopyOutputKey{Path: "/"}: {Path: "/", ContentLength: "0.00 B"}, - } - } - // Scale up from service to object - srcObj := CreateResource[ObjectResourceManager](svm, srcService, ResourceDefinitionObject{ - ObjectName: pointerTo("test"), - Body: body, - }) - expectedObjects[AzCopyOutputKey{Path: "test"}] = cmd.AzCopyListObject{Path: "test", ContentLength: "1.00 KiB"} - - stdout, _ := RunAzCopy( - svm, - AzCopyCommand{ - Verb: AzCopyVerbList, - Targets: []ResourceManager{ - srcObj.Parent().(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ - SASTokenOptions: GenericServiceSignatureValues{ - ContainerName: srcObj.ContainerName(), - Permissions: (&blobsas.ContainerPermissions{Read: true, List: true}).String(), - }, - }), - }, - Flags: ListFlags{}, - }) - - ValidateListOutput(svm, stdout, expectedObjects, nil) jobsListOutput, _ := RunAzCopy( svm, @@ -56,7 +18,8 @@ func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { Verb: AzCopyVerbJobsList, PositionalArgs: []string{"list"}, Stdout: &AzCopyParsedJobsListStdout{}, + Flags: ListFlags{}, }) - ValidateJobsListOutput(svm, jobsListOutput, 1) + ValidateJobsListOutput(svm, jobsListOutput, 0) fmt.Println("stdout Output: ", jobsListOutput) } From 25c1e940460b4aa6659d68c3919cd490e573057b Mon Sep 17 00:00:00 2001 From: Dhanashree Phulkar Date: Wed, 18 Sep 2024 10:57:00 +0530 Subject: [PATCH 513/513] removed the fmt statement --- e2etest/zt_newe2e_jobs_list_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index 135ee3c59..b64bd775f 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -1,9 +1,5 @@ package e2etest -import ( - "fmt" -) - func init() { suiteManager.RegisterSuite(&JobsListSuite{}) } @@ -21,5 +17,4 @@ func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { Flags: ListFlags{}, }) ValidateJobsListOutput(svm, jobsListOutput, 0) - fmt.Println("stdout Output: ", jobsListOutput) }