[buildkite] Add Windows packaging and platform support tests to perio…

…dic pipeline (#98072) (#98328) (cherry picked from commit dfadca8)
elastic · Aug 9, 2023 · a813d01 · a813d01
1 parent 69e92fc
commit a813d01
Show file tree

Hide file tree

Showing 8 changed files with 265 additions and 27 deletions.
diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat
@@ -0,0 +1,21 @@
+@ECHO OFF
+
+FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i
+
+SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA%
+SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16
+
+SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+
+(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties")
+
+set WORKSPACE=%cd%
+set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
+set COMPOSE_HTTP_TIMEOUT=120
+set JOB_BRANCH=%BUILDKITE_BRANCH%
+
+set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
+set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
+
+exit /b 0
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -50,3 +50,50 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: packaging-tests-windows
+    steps:
+      - label: "{{matrix.image}} / packaging-tests-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+        agents:
+          provider: gcp
+          image: family/brian-elasticsearch-{{matrix.image}}
+          imageProject: elastic-images-qa
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env: {}
+  - group: platform-support-windows
+    steps:
+      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 360
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/brian-elasticsearch-{{matrix.image}}
+          imageProject: elastic-images-qa
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
diff --git a/.buildkite/scripts/run-script.ps1 b/.buildkite/scripts/run-script.ps1
@@ -0,0 +1,154 @@
+# Usage: .buildkite/scripts/run-script.ps1 <script-or-simple-command>
+# Example: .buildkite/scripts/run-script.ps1 bash .buildkite/scripts/tests.sh
+# Example: .buildkite/scripts/run-script.ps1 .buildkite/scripts/other-tests.ps1
+#
+# NOTE: Apparently passing arguments in powershell is a nightmare, so you shouldn't do it unless it's really simple. Just use the wrapper to call a script instead.
+# See: https://stackoverflow.com/questions/6714165/powershell-stripping-double-quotes-from-command-line-arguments
+# and: https://github.com/PowerShell/PowerShell/issues/3223#issuecomment-487975049
+#
+# See here: https://github.com/buildkite/agent/issues/2202
+# Background processes after the buildkite step script finishes causes the job to hang.
+# So, until this is fixed/changed in buildkite-agent (if ever), we can use this wrapper.
+
+# This wrapper:
+# - Creates a Windows job object (which is like a process group)
+# - Sets JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, which means that when the job object is closed, all processes in the job object are killed
+# - Starts running your given script, and assigns it to the job object
+# - Now, any child processes created by your script will also end up in the job object
+# - Waits for your script (and only your script, not child processes) to finish
+# - Closes the job object, which kills all processes in the job object (including any leftover child processes)
+# - Exits with the exit status from your script
+
+Add-Type -TypeDefinition @'
+using Microsoft.Win32.SafeHandles;
+using System;
+using System.ComponentModel;
+using System.Runtime.InteropServices;
+
+public class NativeMethods
+{
+    public enum JOBOBJECTINFOCLASS
+    {
+        AssociateCompletionPortInformation = 7,
+        BasicLimitInformation = 2,
+        BasicUIRestrictions = 4,
+        EndOfJobTimeInformation = 6,
+        ExtendedLimitInformation = 9,
+        SecurityLimitInformation = 5,
+        GroupInformation = 11
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct JOBOBJECT_BASIC_LIMIT_INFORMATION
+    {
+        public Int64 PerProcessUserTimeLimit;
+        public Int64 PerJobUserTimeLimit;
+        public UInt32 LimitFlags;
+        public UIntPtr MinimumWorkingSetSize;
+        public UIntPtr MaximumWorkingSetSize;
+        public UInt32 ActiveProcessLimit;
+        public Int64 Affinity;
+        public UInt32 PriorityClass;
+        public UInt32 SchedulingClass;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct IO_COUNTERS
+    {
+        public UInt64 ReadOperationCount;
+        public UInt64 WriteOperationCount;
+        public UInt64 OtherOperationCount;
+        public UInt64 ReadTransferCount;
+        public UInt64 WriteTransferCount;
+        public UInt64 OtherTransferCount;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION
+    {
+        public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation;
+        public IO_COUNTERS IoInfo;
+        public UIntPtr ProcessMemoryLimit;
+        public UIntPtr JobMemoryLimit;
+        public UIntPtr PeakProcessMemoryUsed;
+        public UIntPtr PeakJobMemoryUsed;
+    }
+
+    [DllImport("Kernel32.dll", EntryPoint = "AssignProcessToJobObject", SetLastError = true)]
+    private static extern bool NativeAssignProcessToJobObject(SafeHandle hJob, SafeHandle hProcess);
+
+    public static void AssignProcessToJobObject(SafeHandle job, SafeHandle process)
+    {
+        if (!NativeAssignProcessToJobObject(job, process))
+            throw new Win32Exception();
+    }
+
+    [DllImport(
+        "Kernel32.dll",
+        CharSet = CharSet.Unicode,
+        EntryPoint = "CreateJobObjectW",
+        SetLastError = true
+    )]
+    private static extern SafeFileHandle NativeCreateJobObjectW(
+        IntPtr lpJobAttributes,
+        string lpName
+    );
+
+    [DllImport("Kernel32.dll", EntryPoint = "CloseHandle", SetLastError = true)]
+    private static extern bool NativeCloseHandle(SafeHandle hJob);
+
+    [DllImport("kernel32.dll")]
+    public static extern bool SetInformationJobObject(
+        SafeHandle hJob,
+        JOBOBJECTINFOCLASS JobObjectInfoClass,
+        IntPtr lpJobObjectInfo,
+        uint cbJobObjectInfoLength
+    );
+
+    private const UInt32 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000;
+
+    public static SafeHandle CreateJobObjectW(string name)
+    {
+        SafeHandle job = NativeCreateJobObjectW(IntPtr.Zero, name);
+        JOBOBJECT_BASIC_LIMIT_INFORMATION info = new JOBOBJECT_BASIC_LIMIT_INFORMATION();
+        info.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+        JOBOBJECT_EXTENDED_LIMIT_INFORMATION extendedInfo =
+            new JOBOBJECT_EXTENDED_LIMIT_INFORMATION();
+        extendedInfo.BasicLimitInformation = info;
+        int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+        IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length);
+        Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false);
+        SetInformationJobObject(
+            job,
+            JOBOBJECTINFOCLASS.ExtendedLimitInformation,
+            extendedInfoPtr,
+            (uint)length
+        );
+        if (job.IsInvalid)
+            throw new Win32Exception();
+        return job;
+    }
+
+    public static void CloseHandle(SafeHandle job)
+    {
+        if (!NativeCloseHandle(job))
+            throw new Win32Exception();
+    }
+}
+
+'@
+
+$guid = [guid]::NewGuid().Guid
+Write-Output "Creating job object with name $guid"
+$job = [NativeMethods]::CreateJobObjectW($guid)
+$process = Start-Process -PassThru -NoNewWindow powershell -ArgumentList "$args"
+[NativeMethods]::AssignProcessToJobObject($job, $process.SafeHandle)
+
+try {
+  Write-Output "Waiting for process $($process.Id) to complete..."
+  $process | Wait-Process
+  Write-Output "Process finished with exit code $($process.ExitCode), terminating job and exiting..."
+} finally {
+  [NativeMethods]::CloseHandle($job)
+  exit $process.ExitCode
+}
diff --git a/.buildkite/scripts/windows-run-gradle.sh b/.buildkite/scripts/windows-run-gradle.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK
diff --git a/.ci/scripts/packaging-test.ps1 b/.ci/scripts/packaging-test.ps1
@@ -12,21 +12,21 @@ $env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA
 $env:JAVA_TOOL_OPTIONS=''
 
 $ErrorActionPreference="Stop"
-$gradleInit = "C:\Users\$env:username\.gradle\init.d\"
+$gradleInit = "$env:USERPROFILE\.gradle\init.d\"
 echo "Remove $gradleInit"
 Remove-Item -Recurse -Force $gradleInit -ErrorAction Ignore
 New-Item -ItemType directory -Path $gradleInit
 echo "Copy .ci/init.gradle to $gradleInit"
 Copy-Item .ci/init.gradle -Destination $gradleInit
 
 [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
-$env:PATH="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH"
+$env:PATH="$env:USERPROFILE\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH"
 $env:JAVA_HOME=$null
-$env:SYSTEM_JAVA_HOME="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA"
+$env:SYSTEM_JAVA_HOME="$env:USERPROFILE\.java\$env:ES_BUILD_JAVA"
 Remove-Item -Recurse -Force \tmp -ErrorAction Ignore
 New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
-& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
+& .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
 
 exit $LastExitCode
diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh
@@ -1,28 +1,35 @@
 #!/bin/bash
-# drop page cache and kernel slab objects on linux
-[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
 rm -Rfv ~/.gradle/init.d
 mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
-if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-   MAX_WORKERS=16
-elif [ -f /proc/cpuinfo ]; then
-   MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
-else
-   if [[ "$OSTYPE" == "darwin"* ]]; then
-      MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-   else
-      echo "Unsupported OS Type: $OSTYPE"
-      exit 1
-   fi
-fi
-if pwd | grep -v -q ^/dev/shm ; then
-   echo "Not running on a ramdisk, reducing number of workers"
-   MAX_WORKERS=$(($MAX_WORKERS*2/3))
-fi
 
-# Export glibc version as environment variable since some BWC tests are incompatible with later versions
-export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+MAX_WORKERS=4
+
+# Don't run this stuff on Windows
+if ! uname -a | grep -q MING; then
+  # drop page cache and kernel slab objects on linux
+  [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    MAX_WORKERS=16
+  elif [ -f /proc/cpuinfo ]; then
+    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+  else
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+    else
+        echo "Unsupported OS Type: $OSTYPE"
+        exit 1
+    fi
+  fi
+  if pwd | grep -v -q ^/dev/shm ; then
+    echo "Not running on a ramdisk, reducing number of workers"
+    MAX_WORKERS=$(($MAX_WORKERS*2/3))
+  fi
+
+  # Export glibc version as environment variable since some BWC tests are incompatible with later versions
+  export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+fi
 
 set -e
 $GRADLEW -S --max-workers=$MAX_WORKERS $@
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -121,12 +121,16 @@ buildScan {
       buildScanPublished { scan ->
         // Attach build scan link as build metadata
         // See: https://buildkite.com/docs/pipelines/build-meta-data
-        ['buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}"].execute()
+        new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}")
+          .start()
+          .waitFor()
 
         // Add a build annotation
         // See: https://buildkite.com/docs/agent/v3/cli-annotate
         def body = """<div class="mb3"><span class="p1 border rounded">${System.getenv('BUILDKITE_LABEL')}</span> :gradle: build ran: <a href="${scan.buildScanUri}"><code>gradle ${gradle.startParameter.taskNames.join(' ')}</code></a></div>"""
-        ['buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body].execute()
+        new ProcessBuilder('buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body)
+          .start()
+          .waitFor()
       }
     } else {
       tag 'LOCAL'

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java
@@ -39,7 +39,7 @@ public abstract class ReaperService implements BuildService<ReaperService.Params
      */
     public void registerPid(String serviceId, long pid) {
         String[] killPidCommand = OS.<String[]>conditional()
-            .onWindows(() -> new String[] { "Taskkill", "/F", "/PID", String.valueOf(pid) })
+            .onWindows(() -> new String[] { "Taskkill", "/F", "/T", "/PID", String.valueOf(pid) })
             .onUnix(() -> new String[] { "kill", "-9", String.valueOf(pid) })
             .supply();
         registerCommand(serviceId, killPidCommand);