Nightly builds for Stack (#4456)

* Add Azure nightly integration support and bindists generation

* Fix failing integration tests

.azure/azure-nightly-template-linux.yml

@@ -9,11 +9,9 @@ jobs:
     matrix:
       stack:
         BUILD: stack
-        STACK_YAML: stack.yaml
     maxParallel: 1
   steps:
   - script: |
-      echo $OS_NAME
       export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
       echo $STACK_ROOT
       mkdir -p ~/.local/bin
@@ -23,103 +21,38 @@ jobs:
       cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" restore stack --base-branch="${BASE_BRANCH}"
       cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" restore stack work --base-branch="${BASE_BRANCH}"
       ./.travis-setup.sh
-      case "$BUILD" in
-        style)
-          export PATH="$(pwd)"/hlint:$PATH
-          ;;
-        cabal)
-          export PATH=$HOME/.local/bin:$HOME/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/happy/1.19.5/bin:/opt/alex/3.1.7/bin:/opt/cabal/$CABALVER/bin:$PATH
-          ;;
-        *)
-          export PATH=$HOME/.local/bin:/opt/ghc/$GHCVER/bin:/opt/happy/1.19.5/bin:/opt/alex/3.1.7/bin:$PATH
-          ;;
-      esac
+      export PATH=$HOME/.local/bin:/opt/ghc/$GHCVER/bin:/opt/happy/1.19.5/bin:/opt/alex/3.1.7/bin:$PATH
       if ! [ "$BUILD" = style ]; then echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"; fi
-         set -ex
-      case "$BUILD" in
-        style)
-          ./etc/scripts/get-hlint.sh
-          ;;
-        cabal)
-          cabal --version
-          cabal update
-          rm -f $HOME/.cabal/bin/stack
-          echo "stack is located at $(which stack)"
-          stack --version
-
-          echo Removing any old dist files
-          rm -f $(stack --stack-yaml=$STACK_YAML path --dist-dir)/stack-*.tar.gz
-
-          echo To avoid custom Cabal setup business, switching temporarily to Simple
-          cp stack.cabal stack.orig-cabal
-          sed 's@build-type\:.*@build-type\: Simple@' < stack.orig-cabal > stack.cabal
-
-          echo Generating new dist with pvp bounds in the cabal file
-          stack --system-ghc --stack-yaml=$STACK_YAML sdist --pvp-bounds=both
-
-          echo Grabbing the newly generated stack.cabal file from the tarball
-          tar xf $(stack --system-ghc --compiler=ghc-$GHCVER path --dist-dir)/stack-*.tar.gz --wildcards --strip-components=1 '*/stack.cabal'
-
-          echo Switching back to Custom build type
-          cp stack.cabal stack.orig-cabal
-          sed 's@build-type\:.*@build-type\: Custom@' < stack.orig-cabal > stack.cabal
-          rm -f stack.orig-cabal
-
-          echo Performing the actual build now that we have the right stack.cabal
-          cabal install --only-dependencies --enable-tests --enable-benchmarks --force-reinstalls --ghc-options=-O0 --reorder-goals --max-backjumps=-1
-          ;;
-        *)
-          stack --no-terminal build Cabal
-          stack --no-terminal test --only-dependencies
-          ;;
-      esac
-      set +ex
-      GHC_OPTIONS="-Werror"
-      if [ $GHCVER = 8.2.1 ]; then GHC_OPTIONS="$GHC_OPTIONS -Wno-missing-home-modules"; fi
       set -ex
-      case "$BUILD" in
-        style)
-          hlint src/
-          hlint src/ --cpp-define=WINDOWS=1
-          hlint test/ --cpp-simple
-          ;;
-        stack)
-          stack --no-terminal test --haddock --no-haddock-deps --ghc-options="$GHC_OPTIONS"
-          ;;
-        pedantic)
-          stack --system-ghc --no-terminal build --pedantic
-          ;;
-        cabal)
-          cabal configure --enable-tests --enable-benchmarks -v2 --ghc-options="-O0 $GHC_OPTIONS"
-          cabal build
-          cabal test
-          cabal check
-          cabal sdist
-          cabal copy
-          cd test/integration
-          true stack setup --compiler=ghc-$GHCVER
-          true stack test --compiler=ghc-$GHCVER
-          cd ../..
-          SRC_TGZ=$(cabal info . | awk '{print $2;exit}').tar.gz
-          (cd dist && cabal install --force-reinstalls "$SRC_TGZ")
-        ;;
-      esac
+      stack --no-terminal install cabal-install
       set +ex
     env:
       OS_NAME: ${{ parameters.os }}
       AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
       AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
     displayName: 'Installation ${{parameters.os}}'
-    continueOnError: ${{ eq(variables['STACK_YAML'], 'stack-nightly.yaml') }}
   - script: |
-      stack install
-      tar czf stack.tar.gz -C $(stack path --local-bin) stack
-      mv stack.tar.gz $(Build.ArtifactStagingDirectory)
+      curl https://nixos.org/nix/install | sh
+      source ~/.profile
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$HOME/.local/bin:$PATH;
+      set -ex
+      stack test --flag stack:integration-tests stack:test:stack-integration-test
+      set +ex
+    displayName: Integration Test
+  - script: |
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$HOME/.local/bin:$PATH;
+      set -ex
+      stack install stack --local-bin-path $(Build.ArtifactStagingDirectory)
+      set +ex
     displayName: Gzip Stack
+    condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-      artifactName: stack-linux-x86_64.tar.gz
+      artifactName: stack-linux-x86_64
+    condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
   - script: |
       export PATH=$HOME/.local/bin:$PATH;
       export AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)";

.azure/azure-nightly-template-osx.yml

@@ -0,0 +1,65 @@
+jobs:
+- job: ${{ parameters.name }}
+  timeoutInMinutes: 180
+  variables:
+  - group: AWS_Keys
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    matrix:
+      stack:
+        BUILD: stack
+    maxParallel: 1
+  steps:
+  - script: |
+      export AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)";
+      export AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)";
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      echo $STACK_ROOT
+      mkdir -p ~/.local/bin
+      curl -f -L "https://github.com/fpco/cache-s3/releases/download/${CACHE_S3_VERSION}/cache-s3-${CACHE_S3_VERSION}-${OS_NAME}-x86_64.tar.gz" -o ~/.local/bin/cache-s3.tar.gz
+      tar xzf ~/.local/bin/cache-s3.tar.gz -C ~/.local/bin
+      export PATH=$HOME/.local/bin:$PATH;
+      cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" restore stack --base-branch="${BASE_BRANCH}"
+      cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" restore stack work --base-branch="${BASE_BRANCH}"
+      ./.travis-setup.sh
+      brew install mercurial
+      export PATH=$HOME/.local/bin:/opt/ghc/$GHCVER/bin:/opt/happy/1.19.5/bin:/opt/alex/3.1.7/bin:$PATH
+      set -ex
+      stack --no-terminal install cabal-install
+      set +ex
+    env:
+      OS_NAME: ${{ parameters.os }}
+    displayName: 'Installation ${{parameters.os}}'
+  - script: |
+      curl https://nixos.org/nix/install | sh
+      . ~/.nix-profile/etc/profile.d/nix.sh
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$HOME/.local/bin:$PATH;
+      set -ex
+      stack test --flag stack:integration-tests stack:test:stack-integration-test
+      set +ex
+    displayName: Integration Test
+  - script: |
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$HOME/.local/bin:$PATH;
+      stack install stack --local-bin-path $(Build.ArtifactStagingDirectory)
+    condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+    displayName: Gzip Stack
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      artifactName: stack-osx
+    condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+  - script: |
+      export PATH=$HOME/.local/bin:$PATH;
+      export AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)";
+      export AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)";
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      if [ "$(Build.SourceBranchName)" = "${BASE_BRANCH}" ]; then
+        cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" save stack;
+      fi;
+      cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" save stack work
+    env:
+      OS_NAME: ${{ parameters.os }}
+    displayName: 'Cache data'

.azure/azure-nightly-template-windows.yml

@@ -0,0 +1,68 @@
+jobs:
+- job: ${{ parameters.name }}
+  timeoutInMinutes: 180
+  variables:
+  - group: AWS_Keys
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    matrix:
+      stack:
+        BUILD: stack
+    maxParallel: 1
+  steps:
+  - powershell: |
+      [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+      Invoke-WebRequest -Uri "https://github.com/fpco/cache-s3/releases/download/$env:CACHE_S3_VERSION/cache-s3-$env:CACHE_S3_VERSION-$env:OS_NAME-x86_64.zip" -OutFile $env:TEMP\cache-s3.zip
+      Expand-Archive $env:TEMP\cache-s3.zip -DestinationPath $env:TEMP\
+    env:
+      OS_NAME: ${{ parameters.os }}
+    displayName: Download binaries
+  - bash: |
+      export AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)";
+      export AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)";
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$PATH:"/C/Program Files/Mercurial/"
+      export TMP=/D/tmp
+      choco install hg -y
+      /tmp/cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" restore stack --base-branch="${BASE_BRANCH}"
+      /tmp/cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" restore stack work --base-branch="${BASE_BRANCH}"
+      curl -sSkL http://www.stackage.org/stack/windows-i386 -o /usr/bin/stack.zip
+      unzip -o /usr/bin/stack.zip -d /usr/bin/
+      stack setup
+      stack --no-terminal install cabal-install
+      stack --version
+    env:
+      OS_NAME: ${{ parameters.os }}
+    displayName: 'Installation ${{parameters.os}}'
+  - bash: |
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$PATH:"/C/Program Files/Mercurial/"
+      export TMP=/D/tmp
+      set -ex
+      stack test --flag stack:integration-tests stack:test:stack-integration-test
+      set +ex
+    displayName: Integration Test
+  - powershell: |
+      $env:STACK_ROOT = "$(Build.SourcesDirectory)\.stack-root"
+      $env:PATH += ";$env:HOME\.local\bin"
+      stack install stack --local-bin-path $(Build.ArtifactStagingDirectory)
+    displayName: Gzip Stack
+    condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      artifactName: stack-windows
+    condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+  - bash: |
+      export AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)";
+      export AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)";
+      export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root;
+      export PATH=$HOME/.local/bin:$PATH;
+      if [ "$(Build.SourceBranchName)" = "${BASE_BRANCH}" ]; then
+        /tmp/cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" save stack;
+      fi;
+      /tmp/cache-s3 --prefix="${CACHE_S3_PREFIX}" --git-branch="$(Build.SourceBranchName)" --suffix="${OS_NAME}" save stack work
+    env:
+      OS_NAME: ${{ parameters.os }}
+    displayName: 'Cache data'

azure-nightly-pipeline.yml

@@ -6,20 +6,20 @@ variables:
   AWS_REGION: "us-east-1"
 
 jobs:
-# - template: ./.azure/azure-osx-template.yml
-#   parameters:
-#     name: macOS
-#     vmImage: macOS-10.13
-#     os: osx
+- template: ./.azure/azure-nightly-template-osx.yml
+  parameters:
+    name: macOS
+    vmImage: macOS-10.13
+    os: osx
 
 - template: ./.azure/azure-nightly-template-linux.yml
   parameters:
     name: Linux
     vmImage: ubuntu-16.04
     os: linux
 
-# - template: ./.azure/azure-windows-template.yml
-#   parameters:
-#    name: Windows
-#    vmImage: vs2017-win2016
-#    os: windows
+- template: ./.azure/azure-nightly-template-windows.yml
+  parameters:
+   name: Windows
+   vmImage: vs2017-win2016
+   os: windows

test/integration/IntegrationSpec.hs

@@ -129,11 +129,18 @@ copyTree toCopy src dst =
   where
     go srcfp = when (toCopy srcfp) $ liftIO $ do
         Just suffix <- return $ stripPrefix src srcfp
-        let dstfp = dst ++ "/" ++ suffix
+        let dstfp = dst </> stripHeadSeparator suffix
         createDirectoryIfMissing True $ takeDirectory dstfp
         createSymbolicLink srcfp dstfp `catch` \(_ :: IOException) ->
             copyFile srcfp dstfp -- for Windows
 
+    stripHeadSeparator :: FilePath -> FilePath
+    stripHeadSeparator [] = []
+    stripHeadSeparator fp@(x:xs) = if isPathSeparator x
+                                   then xs
+                                   else fp
+
+
 toCopyRoot :: FilePath -> Bool
 toCopyRoot srcfp = any (`isSuffixOf` srcfp)
     -- FIXME command line parameters to control how many of these get

test/integration/tests/4095-utf8-pure-nix/Main.hs

@@ -2,5 +2,8 @@ import StackTest
 
 main :: IO ()
 main = do
-  stack ["build", "--nix-pure"]
-  stack ["exec", "--nix-pure", "ShowUnicode"]
+  if isWindows
+     then logInfo "Disabled on Windows as Nix is not currently supported on Windows."
+     else do
+       stack ["build", "--nix-pure"]
+       stack ["exec", "--nix-pure", "ShowUnicode"]

test/integration/tests/4105-test-coverage-of-internal-lib/Main.hs

@@ -6,7 +6,7 @@ main :: IO ()
 main = do
   stack ["clean"]
   stack ["build"]
-  res <- getCoverageLines . snd <$> stackStderr ["test", "--coverage"]
+  res <- getCoverageLines . snd <$> stackStderr ["test", "--coverage", "--color", "never"]
   case res of
     _:exprs:_ -> unless ("2/2" `isInfixOf` exprs) testFail
     _ -> testFail

test/integration/tests/4220-freeze-command/Main.hs

@@ -1,23 +1,29 @@
 import Control.Monad (unless)
 import StackTest
+import Data.Set
+import Data.List (dropWhileEnd)
+import Data.Char (isSpace)
 
 main :: IO ()
 main = do
   stackCheckStdout ["freeze"] $ \stdOut -> do
-    let expected = unlines
-          [ "resolver:"
-          , "  size: 527165"
-          , "  url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/11/19.yaml"
-          , "  sha256: 0116ad1779b20ad2c9d6620f172531f13b12bb69867e78f4277157e28865dfd4"
-          , "extra-deps:"
-          , "- hackage: a50-0.5@sha256:b8dfcc13dcbb12e444128bb0e17527a2a7a9bd74ca9450d6f6862c4b394ac054,1491"
-          , "  pantry-tree:"
-          , "    size: 409"
-          , "    sha256: a7c6151a18b04afe1f13637627cad4deff91af51d336c4f33e95fc98c64c40d3"
-          ]
-    unless (stdOut == expected) $
-      error $ concat [ "Expected: "
-                   , show expected
-                   , "\nActual: "
-                   , show stdOut
+    let contents = fromList [
+                    "resolver:",
+                    "size: 527165",
+                    "url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/11/19.yaml",
+                    "sha256: 0116ad1779b20ad2c9d6620f172531f13b12bb69867e78f4277157e28865dfd4",
+                    "extra-deps:",
+                    "pantry-tree:",
+                    "hackage: a50-0.5@sha256:b8dfcc13dcbb12e444128bb0e17527a2a7a9bd74ca9450d6f6862c4b394ac054,1491",
+                    "size: 409",
+                    "sha256: a7c6151a18b04afe1f13637627cad4deff91af51d336c4f33e95fc98c64c40d3"
                    ]
+        isLeadingYamlSymbol c = c == '-'
+        trim str = dropWhileEnd isSpace $ dropWhile (\x -> isSpace x || isLeadingYamlSymbol x) str
+    let stdOutLines = fromList $ Prelude.map trim (lines stdOut)
+    unless (stdOutLines == contents) $
+      error $ concat [ "Expected: "
+                     , show contents
+                     , "\nActual: "
+                     , show stdOutLines
+                     ]