From 1d3e985e4f1b0d52adf1a8eba7a4c0a092938cfb Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Mon, 8 Jun 2020 22:46:18 -0700 Subject: [PATCH] [vcpkg] Fix OSX CI by ensuring the downloads directory exists (#11839) * [vcpkg] Remove do-nothing Set-Content from Windows azure-pipelines.yml. * [vcpkg] Fix OSX CI by ensuring the downloads directory exists in advance, and extract common command line parameters with powershell splatting. * [tensorflow-cc] Prevent hang building tensorflow-cc asking to configure iOS. * Skip ignition-msgs5:x64-osx --- ports/tensorflow-cc/CONTROL | 4 +-- ports/tensorflow-cc/portfile.cmake | 7 ++-- scripts/azure-pipelines/clean-tombstones.yml | 20 ++++------- .../azure-pipelines/linux/azure-pipelines.yml | 13 +++++-- .../azure-pipelines/osx/azure-pipelines.yml | 26 ++++++++------ .../osx/initialize-environment.sh | 8 ----- .../windows/azure-pipelines.yml | 16 ++++++--- scripts/ci.baseline.txt | 2 +- toolsrc/src/vcpkg/commands.ciclean.cpp | 36 +++++++++++-------- 9 files changed, 73 insertions(+), 59 deletions(-) delete mode 100755 scripts/azure-pipelines/osx/initialize-environment.sh diff --git a/ports/tensorflow-cc/CONTROL b/ports/tensorflow-cc/CONTROL index 8bcc1cc35999cb..60c30234666f35 100644 --- a/ports/tensorflow-cc/CONTROL +++ b/ports/tensorflow-cc/CONTROL @@ -1,5 +1,5 @@ Source: tensorflow-cc -Version: 1.14-1 +Version: 1.14-2 Description: Library for computation using data flow graphs for scalable machine learning Build-Depends: c-ares -Supports: !x86 \ No newline at end of file +Supports: !x86 diff --git a/ports/tensorflow-cc/portfile.cmake b/ports/tensorflow-cc/portfile.cmake index aebdf8d656e362..8d2c187b057b86 100644 --- a/ports/tensorflow-cc/portfile.cmake +++ b/ports/tensorflow-cc/portfile.cmake @@ -1,9 +1,7 @@ -include(vcpkg_common_functions) - message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.") if (VCPKG_TARGET_ARCHITECTURE STREQUAL x86) - message(FATAL_ERROR "TensorFlow does not support 32bit system.") + message(FATAL_ERROR "TensorFlow does not support 32bit systems.") endif() vcpkg_from_github( @@ -76,6 +74,7 @@ set(ENV{TF_NCCL_VERSION} 2.3) set(ENV{NCCL_INSTALL_PATH} "") set(ENV{CC_OPT_FLAGS} "/arch:AVX") set(ENV{TF_NEED_CUDA} 0) +set(ENV{TF_CONFIGURE_IOS} 0) message(STATUS "Configuring TensorFlow") @@ -118,4 +117,4 @@ file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tenso file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright) file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc) -file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake) \ No newline at end of file +file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake) diff --git a/scripts/azure-pipelines/clean-tombstones.yml b/scripts/azure-pipelines/clean-tombstones.yml index 9ecee265d091e8..dc601bcfc4805e 100644 --- a/scripts/azure-pipelines/clean-tombstones.yml +++ b/scripts/azure-pipelines/clean-tombstones.yml @@ -14,27 +14,19 @@ jobs: displayName: 'Initialize Environment' inputs: filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1' - - script: rmdir /s /q archives\fail - displayName: 'Delete archives\fail' + - script: rmdir /s /q W:\fail + displayName: 'Delete W:\fail' - job: linux displayName: 'Clean Linux Tombstones' pool: $(linux-pool) steps: - - task: Bash@3 - displayName: 'Initialize Environment' - inputs: - filePath: scripts/azure-pipelines/linux/initialize-environment.sh - - bash: rm -rf archives/fail - displayName: 'Delete archives/fail' + - bash: rm -rf /archives/fail + displayName: 'Delete /archives/fail' - job: osx displayName: 'Clean MacOS Tombstones' pool: name: vcpkgAgentPool demands: Agent.OS -equals Darwin steps: - - task: Bash@3 - displayName: 'Initialize Environment' - inputs: - filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh' - - bash: rm -rf archives/fail - displayName: 'Delete archives/fail' + - bash: rm -rf /Users/vagrant/Data/archives/fail + displayName: 'Delete /Users/vagrant/Data/archives/fail' diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 83a63491937ac5..1e2a653e037674 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -33,11 +33,20 @@ jobs: - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - powershell: | - ./vcpkg x-ci-clean --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages + $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads' + $commonArgs = @( + '--binarycaching', + '--x-binarysource=clear;files,/archives,upload', + '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees', + '--x-install-root=/mnt/vcpkg-ci/install', + '--x-packages-root=/mnt/vcpkg-ci/packages' + ) + + ./vcpkg x-ci-clean @commonArgs $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" mkdir $(System.ArtifactsDirectory)/xml-results $env:HOME = '/home/agent' - ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages --downloads-root=/mnt/vcpkg-ci/downloads + ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs displayName: '** Test Modified Ports **' - task: PowerShell@2 displayName: 'Analyze results and prepare test logs' diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index a73a102a4b09c9..a24ed7d9506de7 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -13,12 +13,8 @@ jobs: steps: - bash: | df -h - displayName: 'Report on Disk Space Before Environment' + displayName: 'Report on Disk Space' condition: always() - - task: Bash@3 - displayName: 'Initialize Environment' - inputs: - filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh' - bash: | brew list autoconf || brew install autoconf brew list automake || brew install automake @@ -26,9 +22,10 @@ jobs: brew list gfortran || brew cask install gfortran displayName: 'Install brew dependencies' - bash: | - df -h - displayName: 'Report on Disk Space After Environment' - condition: always() + sudo mkdir /Users/vagrant/Data/downloads || 0 + sudo chmod 777 /Users/vagrant/Data/downloads || 0 + exit 0 + displayName: 'Create /Users/vagrant/Data/downloads' - task: Bash@3 displayName: 'Build vcpkg' inputs: @@ -39,10 +36,19 @@ jobs: - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - powershell: | - ./vcpkg x-ci-clean --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads + $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads' + $commonArgs = @( + '--binarycaching', + '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload', + '--x-buildtrees-root=/Users/vagrant/Data/buildtrees', + '--x-install-root=/Users/vagrant/Data/installed', + '--x-packages-root=/Users/vagrant/Data/packages' + ) + + ./vcpkg x-ci-clean @commonArgs $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" mkdir $(System.ArtifactsDirectory)/xml-results - ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/Users/vagrant/Data/archives`,upload --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads + ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs displayName: '** Test Modified Ports **' - task: PowerShell@2 displayName: 'Analyze results and prepare test logs' diff --git a/scripts/azure-pipelines/osx/initialize-environment.sh b/scripts/azure-pipelines/osx/initialize-environment.sh deleted file mode 100755 index e947906685b898..00000000000000 --- a/scripts/azure-pipelines/osx/initialize-environment.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT - -# Sets up the environment for MacOS runs of vcpkg CI - -#delete downloaded files that have not been used in 7 days -find ~/Data/downloads/ -maxdepth 1 -type f ! -atime 7 -exec rm -f {} \; diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index f2b14657b5ad70..24089b674dac41 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -23,17 +23,25 @@ jobs: set VCPKG_DOWNLOADS=D:\downloads .\bootstrap-vcpkg.bat - powershell: | - .\vcpkg.exe x-ci-clean --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads + $env:VCPKG_DOWNLOADS='D:\downloads' + $commonArgs = @( + '--binarycaching', + '--x-binarysource=clear;files,W:\,upload', + '--x-buildtrees-root=E:\buildtrees', + '--x-install-root=E:\install', + '--x-packages-root=E:\packages' + ) + + .\vcpkg.exe x-ci-clean @commonArgs $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt mkdir $(System.ArtifactsDirectory)\xml-results # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed. # Install them so the CI succeeds: if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) { - .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads + .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs } - .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads - Set-Content -Path 'run_ci.cmd' -Value $command -Encoding ASCII + .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs displayName: '** Test Modified Ports **' - task: PowerShell@2 displayName: 'Analyze results and prepare test logs' diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 13a6e9943baa5b..df31d14896496d 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -621,7 +621,7 @@ ignition-msgs5:x64-linux=fail ignition-msgs5:arm64-windows=fail ignition-msgs5:arm-uwp=fail ignition-msgs5:x64-uwp=fail -ignition-msgs5:x64-osx=fail +ignition-msgs5:x64-osx=skip imgui-sfml:x64-linux=ignore intel-ipsec:arm64-windows=fail intel-ipsec:arm-uwp=fail diff --git a/toolsrc/src/vcpkg/commands.ciclean.cpp b/toolsrc/src/vcpkg/commands.ciclean.cpp index 9c69d1931ca308..74b9b35ab53dc7 100644 --- a/toolsrc/src/vcpkg/commands.ciclean.cpp +++ b/toolsrc/src/vcpkg/commands.ciclean.cpp @@ -2,31 +2,39 @@ #include #include +#include #include #include using namespace vcpkg; -namespace vcpkg::Commands::CIClean -{ - void perform_and_exit(const VcpkgCmdArguments&, const VcpkgPaths& paths) +namespace { + void clear_directory(Files::Filesystem& fs, const fs::path& target) { - auto& fs = paths.get_filesystem(); - if (fs.is_directory(paths.buildtrees)) + using vcpkg::System::print2; + if (fs.is_directory(target)) { - fs.remove_all_inside(paths.buildtrees, VCPKG_LINE_INFO); + print2("Clearing contents of ", target.u8string(), "\n"); + fs.remove_all_inside(target, VCPKG_LINE_INFO); } - - if (fs.is_directory(paths.installed)) - { - fs.remove_all_inside(paths.installed, VCPKG_LINE_INFO); - } - - if (fs.is_directory(paths.packages)) + else { - fs.remove_all_inside(paths.packages, VCPKG_LINE_INFO); + print2("Skipping clearing contents of ", target.u8string(), " because it was not a directory\n"); } + } +} +namespace vcpkg::Commands::CIClean +{ + void perform_and_exit(const VcpkgCmdArguments&, const VcpkgPaths& paths) + { + using vcpkg::System::print2; + auto& fs = paths.get_filesystem(); + print2("Starting vcpkg CI clean\n"); + clear_directory(fs, paths.buildtrees); + clear_directory(fs, paths.installed); + clear_directory(fs, paths.packages); + print2("Completed vcpkg CI clean\n"); Checks::exit_success(VCPKG_LINE_INFO); } }