i#2861 cronbuilds: Build cronbuilds as official packages (#4089)

Switches to a separate invocation of package.cmake for cronbuilds, to make them look like release packages. Cronbuilds on Travis and Appveyor no longer run the same jobs as a regular build but instead run one package.cmake job. Checks out Dr. Memory to include it in the package. Does a shallow clone, assuming 250 is enough for the embedded DR. Excludes drmemory/ from vera checks. Removes the "package" parameter from non-cron builds which should make them a little faster (xref i#4059). There are still further decisions and actions but now the cronbuilds have the same content as past manual builds. Issue: #2861, #4059
DynamoRIO · Feb 9, 2020 · 47ea6ac · 47ea6ac
1 parent 55172f9
commit 47ea6ac
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 27 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2017-2019 Google, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Google, Inc.  All rights reserved.
 # **********************************************************
 
 # Redistribution and use in source and binary forms, with or without
@@ -90,6 +90,17 @@ install:
   ##################################################
   # XXX i#2145: point at Qt5 for testing drgui build.
 
+  # Include Dr. Memory in cronbuild packages.
+  # We do shallow clones and assume DrM will update its DR at least once
+  # every 250 DR commits.
+  - ps: >-
+      If ($env:APPVEYOR_REPO_TAG -Match "true") {
+        git clone --depth=2 https://github.com/DynamoRIO/drmemory.git drmemory
+        cd drmemory
+        git submodule update --init --depth 250
+        cd ..
+      }
+
 before_build:
   - if "%configuration%"=="2017" call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars32.bat"
   - cd c:\projects\dynamorio
@@ -99,16 +110,15 @@ build_script:
   - cd build
   - echo %PATH%
   # The perl in c:\perl can't open a pipe so we use cygwin perl.
-  # XXX i#1967: can we pass "package" only when deploying to save time?
-  - c:\cygwin\bin\perl ../suite/runsuite_wrapper.pl travis use_ninja package %EXTRA_ARGS%
+  - c:\cygwin\bin\perl ../suite/runsuite_wrapper.pl travis use_ninja %EXTRA_ARGS%
 
 # Automated deployment of builds to GitHub Releases.
 # We rely on a Travis cron job to push a tag to the repo which then
 # triggers this deployment.
-# We disable test running for these package builds in runsuite.cmake by
-# looking for $APPVEYOR_REPO_TAG=="true".
+# We switch to package.cmake for these builds in runsuite_wrapper.pl by looking
+# for $APPVEYOR_REPO_TAG=="true".
 artifacts:
-  - path: 'build\build_*\DynamoRIO*.zip'
+  - path: 'build\DynamoRIO*.zip'
     name: DR.zip
     type: zip
 deploy:

diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2016-2019 Google, Inc.  All rights reserved.
+# Copyright (c) 2016-2020 Google, Inc.  All rights reserved.
 # **********************************************************
 
 # Redistribution and use in source and binary forms, with or without
@@ -55,23 +55,28 @@ language:
 jobs:
   include:
     # 32-bit Linux build with gcc and run tests:
-    - os: linux
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
       compiler: gcc
-      env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=yes EXTRA_ARGS="32_only package"
+      env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS="32_only"
     # 64-bit Linux build with gcc and run tests:
-    - os: linux
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
       compiler: gcc
-      env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=yes EXTRA_ARGS="64_only package"
+      env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS="64_only"
     # AArchXX cross-compile with gcc, no tests:
-    - os: linux
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
       compiler: gcc
       env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=yes DEPLOY=no
     # Android ARM cross-compile with gcc, no tests:
-    - os: linux
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
       compiler: gcc
       env: DYNAMORIO_CROSS_ANDROID_ONLY=yes DEPLOY=no DYNAMORIO_ANDROID_TOOLCHAIN='/tmp/android-gcc-arm-ndk-10e'
     # 32-bit Linux build with clang, no tests (runsuite.cmake disables the tests):
-    - os: linux
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
       compiler: clang
       # We need clang 9.0 to avoid a compiler bug: i#3989.
       env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS=32_only CC=clang-9 && CXX=clang++-9
@@ -85,7 +90,8 @@ jobs:
           - clang-9
     # 64-bit Linux build with clang, no tests (runsuite.cmake disables the tests),
     # install and require clang-format:
-    - os: linux
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
       compiler: clang
       # We need clang 9.0 to avoid a compiler bug: i#3989.
       env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS="64_only require_format" CC=clang-9 && CXX=clang++-9
@@ -99,11 +105,18 @@ jobs:
           - clang-9
           - clang-format-6.0
     # 32-bit OSX build with clang and run tests:
-    - os: osx
+    - if: env(TRAVIS_EVENT_TYPE) != cron
+      os: osx
       # gcc on Travis claims to not be CMAKE_COMPILER_IS_GNUCC so we only run clang.
       compiler: clang
       # We do not have 64-bit support on OSX yet (i#1979).
       env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS=32_only
+    # Package builds.
+    - if: env(TRAVIS_EVENT_TYPE) = cron
+      os: linux
+      compiler: gcc
+      env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=yes
+    # TODO i#3002: Build packages for AArchXX.
 
 # For C/C++ there is no default install, so we set "install", not "before_install".
 install:
@@ -138,13 +151,21 @@ install:
           ln -sf arm-linux-androideabi-ld.bfd /tmp/android-gcc-arm-ndk-10e/bin/arm-linux-androideabi-ld
           cd -
       fi
+  # Include Dr. Memory in cronbuild packages.
+  # We do shallow clones and assume DrM will update its DR at least once
+  # every 250 DR commits.
+  - >
+      if [[ "`uname`" == "Linux" && $TRAVIS_EVENT_TYPE == cron ]]; then
+          git clone --depth=2 https://github.com/DynamoRIO/drmemory.git drmemory
+          cd drmemory && git submodule update --init --depth 250 && cd ..
+      fi
 
 script:
   - suite/runsuite_wrapper.pl travis $EXTRA_ARGS
 
 # For now we create packages as part of each (enabled) job.
-# We disable test running for these package builds in runsuite.cmake by
-# looking for $TRAVIS_EVENT_TYPE=="cron".
+# We switch to package.cmake for these builds in runsuite_wrapper.pl by looking
+# for $TRAVIS_EVENT_TYPE=="cron".
 # Longer-term we may want to use package.cmake instead and even make official
 # builds on Travis (i#2861).
 before_deploy:
@@ -161,12 +182,12 @@ deploy:
   api_key:
     secure: V3kgcRiwijjpmcSuVio1+/oZ8cqJGaVlL42hN0w/jjO6LoELy2kknT5h80H7wMVKpZnMg+2v/yWj5hawlrwh8nCS51lYllPHN7K+ivzkyJ3R4cp1WAzL56vnYFYz1/twYpeS10Zl6JL6wt788WcibpShMOIlAnXnm1kU9BBVtYE=
   file_glob: true
-  file: "build*/DynamoRIO*.tar.gz"
+  file: "DynamoRIO*.tar.gz"
   skip_cleanup: true
   # The name must just be the tag in order to match Appveyor.
   name: $GIT_TAG
   # This body is clobbered by Appveyor.
-  body: "Auto-generated periodic build (Travis build $TRAVIS_BUILD_NUMBER).  Unlike official release builds, Dr. Memory is not included in this build, and i686 is separated from x86_64 rather than combined in one package."
+  body: "Auto-generated periodic build (Travis build $TRAVIS_BUILD_NUMBER)."
   on:
     repo: DynamoRIO/dynamorio
     branch: master

diff --git a/suite/runsuite_wrapper.pl b/suite/runsuite_wrapper.pl
@@ -58,6 +58,15 @@
     }
 }
 
+my $osdir = $mydir;
+if ($^O eq 'cygwin') {
+    # CMake is native Windows so pass it a Windows path.
+    # We use the full path to cygpath as git's cygpath is earlier on
+    # the PATH for AppVeyor and it fails.
+    $osdir = `/usr/bin/cygpath -wi \"$mydir\"`;
+    chomp $osdir;
+}
+
 # We tee to stdout to provide incremental output and avoid the 10-min
 # no-output timeout on Travis.
 my $res = '';
@@ -71,6 +80,13 @@
         $res .= $_;
     }
     close(CHILD);
+} elsif ($ENV{'TRAVIS_EVENT_TYPE'} eq 'cron' ||
+         $ENV{'APPVEYOR_REPO_TAG'} eq 'true') {
+    # A package build.
+    my $def_args = "build=1;invoke=${osdir}/../drmemory/package.cmake;drmem_only";
+    $args =~ s/^,/;/;
+    system("ctest -VV -S \"${osdir}/../make/package.cmake,${def_args}${args}\" 2>&1");
+    exit 0;
 } else {
     # We have no way to access the log files, so we can -VV to ensure
     # we can diagnose failures, but it makes for a large online result
@@ -80,13 +96,8 @@
     my $verbose = "-V";
     if ($^O eq 'cygwin') {
         $verbose = "-VV";
-        # CMake is native Windows so pass it a Windows path.
-        # We use the full path to cygpath as git's cygpath is earlier on
-        # the PATH for AppVeyor and it fails.
-        $mydir = `/usr/bin/cygpath -wi \"$mydir\"`;
-        chomp $mydir;
     }
-    system("ctest --output-on-failure ${verbose} -S \"${mydir}/runsuite.cmake${args}\" 2>&1");
+    system("ctest --output-on-failure ${verbose} -S \"${osdir}/runsuite.cmake${args}\" 2>&1");
     exit 0;
 }
 

diff --git a/third_party/vera++/use_vera++.cmake b/third_party/vera++/use_vera++.cmake
@@ -113,7 +113,9 @@ function(add_vera_targets_for_dynamorio)
         NOT s MATCHES "/third_party/" AND
         # Somehow on Travis vera checks build-dir files.
         NOT s MATCHES "/build_" AND
-        NOT s MATCHES "/install/")
+        NOT s MATCHES "/install/" AND
+        # We check out drmemory for package builds.
+        NOT s MATCHES "/drmemory/")
       get_filename_component(d ${s} PATH)
       if(NOT "${d}" STREQUAL "${currentDir}")
         # this is a new dir - lets generate everything needed for the previous dir