jenkins: add Windows scripts

PR-URL: #2099
Reviewed-By: Rod Vagg <rod@vagg.org>
Reviewed-By: Richard Lau <riclau@uk.ibm.com>
Reviewed-By: Christian Clauss <cclauss@me.com>

jenkins/scripts/ci-node-version.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -ex
+
+rm -rf src node_version.h.tar || true
+
+source env.properties
+
+set +x
+export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $JENKINS_TMP_KEY"
+set -x
+
+# Retry once
+git archive --format=tar --remote="$TEMP_REPO" $TEMP_BRANCH src/node_version.h -o node_version.h.tar ||
+git archive --format=tar --remote="$TEMP_REPO" $TEMP_BRANCH src/node_version.h -o node_version.h.tar
+
+tar xvf node_version.h.tar
+
+# Same command also used by iojs+release
+export NODEJS_MAJOR_VERSION=$(cat src/node_version.h | grep "#define NODE_MAJOR_VERSION" | awk '{ print $3}')
+
+echo NODEJS_MAJOR_VERSION=$NODEJS_MAJOR_VERSION >> env.properties

jenkins/scripts/windows/.gitattributes

@@ -0,0 +1,2 @@
+/**/*.bat text eol=crlf
+/**/*.cmd text eol=crlf

jenkins/scripts/windows/ci-cleanup.cmd

@@ -0,0 +1,19 @@
+:: Processes that can and have interfered with CI in the past
+taskkill /t /f /fi "IMAGENAME eq node.exe"
+taskkill /t /f /fi "IMAGENAME eq cctest.exe"
+taskkill /t /f /fi "IMAGENAME eq run-tests.exe"
+taskkill /t /f /fi "IMAGENAME eq msbuild.exe"
+taskkill /t /f /fi "IMAGENAME eq mspdbsrv.exe"
+taskkill /t /f /fi "IMAGENAME eq yes.exe"
+taskkill /t /f /fi "IMAGENAME eq python.exe"
+taskkill /t /f /fi "IMAGENAME eq node-copy.exe"
+
+:: rm is better for very long paths
+rm -rfv node/test/tmp*
+
+:: Attempt to recover the worker if a run is aborted during git operations
+if exist build\.git\index.lock rd /s /q build\.git
+if exist node\.git\index.lock rd /s /q node\.git
+
+:: Don't fail - clear errorlevel
+ver > nul

jenkins/scripts/windows/ci-run.cmd

@@ -0,0 +1,31 @@
+set "PROMPT=> "
+echo on
+
+:: Information
+git --version
+
+:: Checkout nodejs/build repository or fork with scripts
+mkdir build
+cd build
+git init
+git clean -dffx
+git remote add GitHub https://github.com/nodejs/build.git
+git fetch --prune --no-tags GitHub
+git fetch --prune --no-tags https://github.com/%SCRIPTS_REPO%.git +refs/heads/%SCRIPTS_BRANCH%:refs/remotes/jenkins_tmp
+if errorlevel 1 exit /b
+git checkout -f refs/remotes/jenkins_tmp
+if errorlevel 1 exit /b
+git reset --hard
+if errorlevel 1 exit /b
+git clean -dffx
+if errorlevel 1 exit /b
+cd %~dp0
+
+:: Clean up before running
+call build\jenkins\scripts\windows\ci-cleanup.cmd
+
+:: Run script
+mkdir node
+cd node
+call ..\build\jenkins\scripts\windows\%1.cmd
+if errorlevel 1 exit /b

jenkins/scripts/windows/compile.cmd

@@ -0,0 +1,30 @@
+:: Use Python 2 up to Node 12
+if %NODEJS_MAJOR_VERSION% leq 12 set "PATH=C:\Python27\;C:\Python27\Scripts;%PATH%"
+
+:: Opt-in for a generating binlog (work with code has https://github.com/nodejs/node/pull/26431/files)
+set "msbuild_args=/binaryLogger:node.binlog"
+
+:: Opt-in for a clcache
+if not defined DISABLE_CLCACHE if exist C:\clcache\dist\clcache_main\clcache_main.exe (
+  set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=60000
+  set CLCACHE_BASEDIR="%WORKSPACE%"
+  set CLCACHE_HARDLINK=1
+  set "msbuild_args=%msbuild_args% /p:CLToolExe=clcache_main.exe /p:CLToolPath=C:\clcache\dist\clcache_main"
+  :: multiproc msbuild doesn't play nice with clcache
+  set NUMBER_OF_PROCESSORS=1
+  :: Ensure cache size
+  C:\clcache\dist\clcache_main\clcache_main.exe -M 20000000000
+  C:\clcache\dist\clcache_main\clcache_main.exe -s
+)
+
+:: Call vcbuild
+if "%nodes:~-4%" == "-x86" (
+  set "VCBUILD_EXTRA_ARGS=x86 %VCBUILD_EXTRA_ARGS%"
+) else (
+  set "VCBUILD_EXTRA_ARGS=x64 %VCBUILD_EXTRA_ARGS%"
+)
+set DEBUG_HELPER=1
+call vcbuild.bat %VCBUILD_EXTRA_ARGS%
+if errorlevel 1 exit /b
+:: echo back on after vcbuild
+echo on

jenkins/scripts/windows/git-checkout.cmd

@@ -0,0 +1,44 @@
+git --version
+git init
+git clean -dffx
+
+:: This is just an optimization step. It's more efficient to fetch as much as possible from GitHub
+git remote add GitHub https://github.com/nodejs/node.git
+git fetch --prune --no-tags GitHub
+if errorlevel 1 echo Problem fetching the main repo.
+
+:: Privately set git over ssh to our TEMP_REPO
+echo off
+for /f "delims=" %%F in ('cygpath -u %JENKINS_TMP_KEY%') do set "GIT_SSH_COMMAND=ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %%F"
+echo on
+
+:: Get TEMP_BRANCH from temp repo, retry once
+git fetch --prune --no-tags %GIT_REPO% +%GIT_REF%:refs/remotes/jenkins_tmp
+if errorlevel 1 (
+  sleep 10
+  git fetch --prune --no-tags %GIT_REPO% +%GIT_REF%:refs/remotes/jenkins_tmp
+  if errorlevel 1 exit /b
+)
+
+:: Branch needs to be in remotes/ for the checkout here to leave a detached HEAD state. If the branch was in heads/, the next run would fail when overwriting.
+git checkout -f refs/remotes/jenkins_tmp
+if errorlevel 1 exit /b
+
+:: Clean up.
+git reset --hard
+if errorlevel 1 exit /b
+git clean -dffx
+if errorlevel 1 exit /b
+
+:: https://github.com/nodejs/node/issues/5094
+for /f %%F in ('git status --porcelain') do (
+  git status
+  echo "Workspace cannot be cleaned, possible problem with committed file names"
+  exit /b 1
+)
+
+:: Check NODEJS_MAJOR_VERSION
+for /f "delims=." %%a in ('python tools\getnodeversion.py') do if not "%%a" == "%NODEJS_MAJOR_VERSION%" (
+  echo "NODEJS_MAJOR_VERSION parameter (%NODEJS_MAJOR_VERSION%) does not match output of tools\getnodeversion.py (%%a)"
+  exit /b 1
+)

jenkins/scripts/windows/node-compile-windows-debug.cmd

@@ -0,0 +1,17 @@
+:: Git
+set GIT_REPO=%TEMP_REPO%
+set GIT_REF=refs/heads/%TEMP_BRANCH%
+call %~dp0git-checkout.cmd
+if errorlevel 1 exit /b
+
+:: Compile
+set "VCBUILD_EXTRA_ARGS=debug"
+call %~dp0compile.cmd
+if errorlevel 1 exit /b
+
+:: Basic sanity check
+:: Test compiling and running a native add-on
+%node_gyp_exe% rebuild --directory=test\addons\hello-world --nodedir="%CD%"
+if errorlevel 1 exit /b
+Debug\node.exe test\addons\hello-world\test.js
+if errorlevel 1 exit /b

jenkins/scripts/windows/node-compile-windows.cmd

@@ -0,0 +1,47 @@
+:: Git
+set GIT_REPO=%TEMP_REPO%
+set GIT_REF=refs/heads/%TEMP_BRANCH%
+call %~dp0git-checkout.cmd
+if errorlevel 1 exit /b
+
+:: Delete binary branch that will be built here if it exists
+git push %TEMP_REPO% :%TEMP_PUSH_BRANCH%-bin-%nodes%
+:: reset errorlevel
+ver > nul
+
+:: Compile
+set "VCBUILD_EXTRA_ARGS=release msi"
+call %~dp0compile.cmd
+if errorlevel 1 exit /b
+
+:: Select files to pack
+set "BINARY_FILES=config.gypi icu_config.gypi Release/node.exe Release/node.lib Release/openssl-cli.exe Release/cctest.exe Release/node.pdb"
+if exist Release\node_test_engine.dll set "BINARY_FILES=%BINARY_FILES% Release/node_test_engine.dll"
+
+:: Setup binary package
+rm -rf binary
+mkdir binary
+tar cavf binary/binary.tar.gz %BINARY_FILES%
+if errorlevel 1 exit /b
+md5sum binary/binary.tar.gz %BINARY_FILES%
+:: make a copy for artifact
+copy binary\binary.tar.gz .
+if errorlevel 1 exit /b
+
+:: Create local temp branch with binary
+git config --replace-all user.name "Node.js Jenkins CI"
+git config --replace-all user.email ci@iojs.org
+git checkout -B %TEMP_PUSH_BRANCH%-bin-%nodes%
+if errorlevel 1 exit /b
+git add binary\binary.tar.gz
+if errorlevel 1 exit /b
+git commit -m "added binaries"
+if errorlevel 1 exit /b
+
+:: Push temp branch
+git push "%TEMP_REPO%" "+%TEMP_PUSH_BRANCH%-bin-%nodes%"
+if errorlevel 1 exit /b
+
+:: Delete local temp branch with binary
+git checkout -f refs/remotes/jenkins_tmp
+git branch -D %TEMP_PUSH_BRANCH%-bin-%nodes%

jenkins/scripts/windows/node-test-binary-windows-js-suites.cmd

@@ -0,0 +1,18 @@
+:: Git
+set COMPILED_BY=%nodes:*-COMPILED_BY-=%
+set GIT_REPO=%TEMP_REPO%
+set GIT_REF=refs/heads/%TEMP_BRANCH%-bin-win-%COMPILED_BY%
+call %~dp0git-checkout.cmd
+if errorlevel 1 exit /b
+
+:: Run tests
+set "test_ci_args=--run=%RUN_SUBSET%,4"
+set VCBUILD_TARGET=test-ci-js
+call %~dp0test.cmd
+if errorlevel 1 exit /b
+
+:: Check existence of test results
+if not exist js-tests.junit.xml (
+  echo "Could not find test.tap or js-tests.junit.xml"
+  exit /b 1
+)

jenkins/scripts/windows/node-test-binary-windows-native-suites.cmd

@@ -0,0 +1,22 @@
+:: Git
+set COMPILED_BY=%nodes:*-COMPILED_BY-=%
+set GIT_REPO=%TEMP_REPO%
+set GIT_REF=refs/heads/%TEMP_BRANCH%-bin-win-%COMPILED_BY%
+call %~dp0git-checkout.cmd
+if errorlevel 1 exit /b
+
+:: Run tests
+set "test_ci_args="
+set VCBUILD_TARGET=test-ci-native
+call %~dp0test.cmd
+if errorlevel 1 exit /b
+
+:: Check existence of test results
+if not exist js-tests.junit.xml (
+  echo "Could not find test.tap or js-tests.junit.xml"
+  exit /b 1
+)
+if not exist cctest.junit.xml (
+  echo "Could not find cctest.tap or cctest.junit.xml"
+  exit /b 1
+)

jenkins/scripts/windows/test.cmd

@@ -0,0 +1,65 @@
+git rev-parse --verify HEAD
+git rev-parse "HEAD^"
+
+:: Extract binary
+md5sum binary/binary.tar.gz
+tar xzvf binary/binary.tar.gz
+if errorlevel 1 exit /b
+if exist out\Release\node.exe ln -s out/Release Release
+md5sum config.gypi icu_config.gypi Release/node.exe Release/node.lib Release/openssl-cli.exe Release/cctest.exe
+
+:: Run the tests
+call :diagnostics Before
+set DEBUG_HELPER=1
+call vcbuild.bat release noprojgen nobuild ignore-flaky %VCBUILD_TARGET%
+set "EXIT_RETURN_VALUE=%errorlevel%"
+ver > nul
+call :diagnostics After
+
+:: Convert test results
+if exist test.tap (
+  tap2junit -i test.tap -o js-tests.junit.xml
+  if errorlevel 1 exit /b
+)
+if exist cctest.tap (
+  tap2junit -i cctest.tap -o cctest.junit.xml
+  if errorlevel 1 exit /b
+)
+
+:: The JUnit Plugin only marks the job as Unstable when it finds any kind of failure, including flaky tests.
+:: We need to use the return code of vcbuild.bat to fail the job when there are real failures.
+exit /b %EXIT_RETURN_VALUE%
+
+
+
+:diagnostics
+echo off
+set "DIAGFILE=c:\jenkins_diagnostics.txt"
+echo. >> %DIAGFILE%
+echo. >> %DIAGFILE%
+echo. >> %DIAGFILE%
+set "TS=%date% %time%"
+echo %TS%
+echo %TS% >> %DIAGFILE%
+echo %1 running vcbuild >> %DIAGFILE%
+echo %BUILD_TAG% >> %DIAGFILE%
+echo %BUILD_URL% >> %DIAGFILE%
+echo %NODE_NAME% >> %DIAGFILE%
+echo. >> %DIAGFILE%
+echo netstat -abno >> %DIAGFILE%
+netstat -abno >> %DIAGFILE% 2>&1
+echo. >> %DIAGFILE%
+echo tasklist /v >> %DIAGFILE%
+tasklist /v >> %DIAGFILE% 2>&1
+echo. >> %DIAGFILE%
+echo tasklist /svc >> %DIAGFILE%
+tasklist /svc >> %DIAGFILE% 2>&1
+echo. >> %DIAGFILE%
+echo wmic path win32_process get Caption,Processid,Commandline >> %DIAGFILE%
+wmic path win32_process get Caption,Processid,Commandline | more >> %DIAGFILE% 2>&1
+mv %DIAGFILE% %DIAGFILE%-OLD
+cat %DIAGFILE%-OLD | sed s/\r//g;s/$/\r/ | tail -c 20000000 > %DIAGFILE%
+rm %DIAGFILE%-OLD
+echo on
+tasklist | grep node
+exit /b 0