From cddccfffc24479789f5744b35fe840b92e3970e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Wed, 30 Oct 2019 06:55:34 +0000 Subject: [PATCH 01/17] jenkins: add Windows to VersionSelectorScript --- jenkins/scripts/VersionSelectorScript.groovy | 42 ++++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/jenkins/scripts/VersionSelectorScript.groovy b/jenkins/scripts/VersionSelectorScript.groovy index 986e4fef9..4f12625a8 100644 --- a/jenkins/scripts/VersionSelectorScript.groovy +++ b/jenkins/scripts/VersionSelectorScript.groovy @@ -1,7 +1,14 @@ +// https://wiki.jenkins.io/display/JENKINS/matrix+groovy+execution+strategy+plugin +// https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html + // Helper closures to make our buildExclusions DSL terse and readable -def gte = { v -> { nodeVersion -> nodeVersion >= v} } -def lt = { v -> { nodeVersion -> nodeVersion < v} } +def lt = { v -> { nodeVersion -> nodeVersion < v } } +def gte = { v -> { nodeVersion -> nodeVersion >= v } } +def ltGte = { vLt, vGte -> { nodeVersion -> lt(vLt)(nodeVersion) || gte(vGte)(nodeVersion) } } +def allVer = { nodeVersion -> true } +def noVer = { nodeVersion -> false } def releaseType = { buildType -> buildType == 'release' } +def testType = { buildType -> buildType == 'test' } def anyType = { buildType -> true } def buildExclusions = [ @@ -38,11 +45,32 @@ def buildExclusions = [ [ /^cross-compiler-armv[67]-gcc-6/, anyType, lt(12) ], // Windows ----------------------------------------------- - [ /vs2013/, anyType, gte(6) ], - [ /vs2015/, anyType, lt(6) ], - [ /vs2015/, anyType, gte(10) ], - [ /vs2017/, anyType, lt(10) ], - [ /vs2019/, anyType, lt(10) ], + // https://github.com/nodejs/build/blob/master/doc/windows-visualstudio-supported-versions.md + // Release Builders - should only match one VS version per Node.js version + [ /vs2013/, releaseType, gte(6) ], + [ /vs2015/, releaseType, ltGte(6, 10) ], + [ /vs2017/, releaseType, ltGte(10, 14) ], + [ /vs2019/, releaseType, lt(14) ], + // VS versions supported to compile Node.js - also matches labels used by test runners + [ /vs2013(-\w+)?$/, testType, gte(6) ], + [ /vs2015(-\w+)?$/, testType, gte(10) ], + [ /vcbt2015(-\w+)?$/, testType, gte(10) ], + [ /vs2017(-\w+)?$/, testType, lt(8) ], + [ /vs2019(-\w+)?$/, testType, lt(13) ], + [ /vs2015-x86$/, testType, gte(10) ], // compile x86 only once + [ /vs2017-x86$/, testType, ltGte(10, 14) ], + [ /vs2019-x86$/, testType, lt(14) ], + // VS versions supported to build add-ons + [ /vs2013-COMPILED_BY/, testType, gte(9) ], + [ /vs2015-COMPILED_BY/, testType, noVer ], + [ /vcbt2015-COMPILED_BY/, testType, noVer ], + [ /vs2017-COMPILED_BY/, testType, lt(8) ], + [ /vs2019-COMPILED_BY/, testType, lt(12) ], + // Exclude some redundant configurations + // https://github.com/nodejs/build/blob/master/doc/node-test-commit-matrix.md + [ /win2008r2.*COMPILED_BY-vs2017/, testType, lt(10) ], // vs2015 runs on win2008 for <10 + [ /win10.*COMPILED_BY-vs2017/, testType, lt(10) ], // vcbt2015 runs on win10 for <10 + [ /win10.*COMPILED_BY-vs2017/, testType, gte(13) ], // vs2019 runs on win10 for >=13 // SmartOS ----------------------------------------------- [ /^smartos15/, anyType, lt(8) ], From bada110c6cf53300436d432dde9e856ad4072c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Tue, 12 Nov 2019 23:25:01 +0000 Subject: [PATCH 02/17] jenkins: add Windows scripts --- jenkins/scripts/ci-node-version.sh | 20 ++++++ jenkins/scripts/windows/.gitattributes | 2 + jenkins/scripts/windows/ci-cleanup.cmd | 19 ++++++ jenkins/scripts/windows/ci-run.cmd | 31 +++++++++ jenkins/scripts/windows/compile.cmd | 30 +++++++++ jenkins/scripts/windows/git-checkout.cmd | 44 +++++++++++++ .../windows/node-compile-windows-debug.cmd | 17 +++++ .../scripts/windows/node-compile-windows.cmd | 47 ++++++++++++++ .../node-test-binary-windows-js-suites.cmd | 18 +++++ ...node-test-binary-windows-native-suites.cmd | 22 +++++++ jenkins/scripts/windows/test.cmd | 65 +++++++++++++++++++ 11 files changed, 315 insertions(+) create mode 100644 jenkins/scripts/ci-node-version.sh create mode 100644 jenkins/scripts/windows/.gitattributes create mode 100644 jenkins/scripts/windows/ci-cleanup.cmd create mode 100644 jenkins/scripts/windows/ci-run.cmd create mode 100644 jenkins/scripts/windows/compile.cmd create mode 100644 jenkins/scripts/windows/git-checkout.cmd create mode 100644 jenkins/scripts/windows/node-compile-windows-debug.cmd create mode 100644 jenkins/scripts/windows/node-compile-windows.cmd create mode 100644 jenkins/scripts/windows/node-test-binary-windows-js-suites.cmd create mode 100644 jenkins/scripts/windows/node-test-binary-windows-native-suites.cmd create mode 100644 jenkins/scripts/windows/test.cmd diff --git a/jenkins/scripts/ci-node-version.sh b/jenkins/scripts/ci-node-version.sh new file mode 100644 index 000000000..f95a0d3f6 --- /dev/null +++ b/jenkins/scripts/ci-node-version.sh @@ -0,0 +1,20 @@ +#!/bin/bash -ex + +rm -rf src node_version.h.tar || true + +source env.properties + +set +x +export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $JENKINS_TMP_KEY" +set -x + +# Retry once +git archive --format=tar --remote="$TEMP_REPO" $TEMP_BRANCH src/node_version.h -o node_version.h.tar || +git archive --format=tar --remote="$TEMP_REPO" $TEMP_BRANCH src/node_version.h -o node_version.h.tar + +tar xvf node_version.h.tar + +# Same command also used by iojs+release +export NODEJS_MAJOR_VERSION=$(cat src/node_version.h | grep "#define NODE_MAJOR_VERSION" | awk '{ print $3}') + +echo NODEJS_MAJOR_VERSION=$NODEJS_MAJOR_VERSION >> env.properties diff --git a/jenkins/scripts/windows/.gitattributes b/jenkins/scripts/windows/.gitattributes new file mode 100644 index 000000000..fe87f5fd8 --- /dev/null +++ b/jenkins/scripts/windows/.gitattributes @@ -0,0 +1,2 @@ +/**/*.bat text eol=crlf +/**/*.cmd text eol=crlf diff --git a/jenkins/scripts/windows/ci-cleanup.cmd b/jenkins/scripts/windows/ci-cleanup.cmd new file mode 100644 index 000000000..ab805d796 --- /dev/null +++ b/jenkins/scripts/windows/ci-cleanup.cmd @@ -0,0 +1,19 @@ +:: Processes that can and have interfered with CI in the past +taskkill /t /f /fi "IMAGENAME eq node.exe" +taskkill /t /f /fi "IMAGENAME eq cctest.exe" +taskkill /t /f /fi "IMAGENAME eq run-tests.exe" +taskkill /t /f /fi "IMAGENAME eq msbuild.exe" +taskkill /t /f /fi "IMAGENAME eq mspdbsrv.exe" +taskkill /t /f /fi "IMAGENAME eq yes.exe" +taskkill /t /f /fi "IMAGENAME eq python.exe" +taskkill /t /f /fi "IMAGENAME eq node-copy.exe" + +:: rm is better for very long paths +rm -rfv node/test/tmp* + +:: Attempt to recover the worker if a run is aborted during git operations +if exist build\.git\index.lock rd /s /q build\.git +if exist node\.git\index.lock rd /s /q node\.git + +:: Don't fail - clear errorlevel +ver > nul diff --git a/jenkins/scripts/windows/ci-run.cmd b/jenkins/scripts/windows/ci-run.cmd new file mode 100644 index 000000000..7274ea58c --- /dev/null +++ b/jenkins/scripts/windows/ci-run.cmd @@ -0,0 +1,31 @@ +set "PROMPT=> " +echo on + +:: Information +git --version + +:: Checkout nodejs/build repository or fork with scripts +mkdir build +cd build +git init +git clean -dffx +git remote add GitHub https://github.com/nodejs/build.git +git fetch --prune --no-tags GitHub +git fetch --prune --no-tags https://github.com/%SCRIPTS_REPO%.git +refs/heads/%SCRIPTS_BRANCH%:refs/remotes/jenkins_tmp +if errorlevel 1 exit /b +git checkout -f refs/remotes/jenkins_tmp +if errorlevel 1 exit /b +git reset --hard +if errorlevel 1 exit /b +git clean -dffx +if errorlevel 1 exit /b +cd %~dp0 + +:: Clean up before running +call build\jenkins\scripts\windows\ci-cleanup.cmd + +:: Run script +mkdir node +cd node +call ..\build\jenkins\scripts\windows\%1.cmd +if errorlevel 1 exit /b diff --git a/jenkins/scripts/windows/compile.cmd b/jenkins/scripts/windows/compile.cmd new file mode 100644 index 000000000..dee51592f --- /dev/null +++ b/jenkins/scripts/windows/compile.cmd @@ -0,0 +1,30 @@ +:: Use Python 2 up to Node 12 +if %NODEJS_MAJOR_VERSION% leq 12 set "PATH=C:\Python27\;C:\Python27\Scripts;%PATH%" + +:: Opt-in for a generating binlog (work with code has https://github.com/nodejs/node/pull/26431/files) +set "msbuild_args=/binaryLogger:node.binlog" + +:: Opt-in for a clcache +if not defined DISABLE_CLCACHE if exist C:\clcache\dist\clcache_main\clcache_main.exe ( + set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=60000 + set CLCACHE_BASEDIR="%WORKSPACE%" + set CLCACHE_HARDLINK=1 + set "msbuild_args=%msbuild_args% /p:CLToolExe=clcache_main.exe /p:CLToolPath=C:\clcache\dist\clcache_main" + :: multiproc msbuild doesn't play nice with clcache + set NUMBER_OF_PROCESSORS=1 + :: Ensure cache size + C:\clcache\dist\clcache_main\clcache_main.exe -M 20000000000 + C:\clcache\dist\clcache_main\clcache_main.exe -s +) + +:: Call vcbuild +if "%nodes:~-4%" == "-x86" ( + set "VCBUILD_EXTRA_ARGS=x86 %VCBUILD_EXTRA_ARGS%" +) else ( + set "VCBUILD_EXTRA_ARGS=x64 %VCBUILD_EXTRA_ARGS%" +) +set DEBUG_HELPER=1 +call vcbuild.bat %VCBUILD_EXTRA_ARGS% +if errorlevel 1 exit /b +:: echo back on after vcbuild +echo on diff --git a/jenkins/scripts/windows/git-checkout.cmd b/jenkins/scripts/windows/git-checkout.cmd new file mode 100644 index 000000000..cf66ba1eb --- /dev/null +++ b/jenkins/scripts/windows/git-checkout.cmd @@ -0,0 +1,44 @@ +git --version +git init +git clean -dffx + +:: This is just an optimization step. It's more efficient to fetch as much as possible from GitHub +git remote add GitHub https://github.com/nodejs/node.git +git fetch --prune --no-tags GitHub +if errorlevel 1 echo Problem fetching the main repo. + +:: Privately set git over ssh to our TEMP_REPO +echo off +for /f "delims=" %%F in ('cygpath -u %JENKINS_TMP_KEY%') do set "GIT_SSH_COMMAND=ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %%F" +echo on + +:: Get TEMP_BRANCH from temp repo, retry once +git fetch --prune --no-tags %GIT_REPO% +%GIT_REF%:refs/remotes/jenkins_tmp +if errorlevel 1 ( + sleep 10 + git fetch --prune --no-tags %GIT_REPO% +%GIT_REF%:refs/remotes/jenkins_tmp + if errorlevel 1 exit /b +) + +:: Branch needs to be in remotes/ for the checkout here to leave a detached HEAD state. If the branch was in heads/, the next run would fail when overwriting. +git checkout -f refs/remotes/jenkins_tmp +if errorlevel 1 exit /b + +:: Clean up. +git reset --hard +if errorlevel 1 exit /b +git clean -dffx +if errorlevel 1 exit /b + +:: https://github.com/nodejs/node/issues/5094 +for /f %%F in ('git status --porcelain') do ( + git status + echo "Workspace cannot be cleaned, possible problem with committed file names" + exit /b 1 +) + +:: Check NODEJS_MAJOR_VERSION +for /f "delims=." %%a in ('python tools\getnodeversion.py') do if not "%%a" == "%NODEJS_MAJOR_VERSION%" ( + echo "NODEJS_MAJOR_VERSION parameter (%NODEJS_MAJOR_VERSION%) does not match output of tools\getnodeversion.py (%%a)" + exit /b 1 +) diff --git a/jenkins/scripts/windows/node-compile-windows-debug.cmd b/jenkins/scripts/windows/node-compile-windows-debug.cmd new file mode 100644 index 000000000..877a12078 --- /dev/null +++ b/jenkins/scripts/windows/node-compile-windows-debug.cmd @@ -0,0 +1,17 @@ +:: Git +set GIT_REPO=%TEMP_REPO% +set GIT_REF=refs/heads/%TEMP_BRANCH% +call %~dp0git-checkout.cmd +if errorlevel 1 exit /b + +:: Compile +set "VCBUILD_EXTRA_ARGS=debug" +call %~dp0compile.cmd +if errorlevel 1 exit /b + +:: Basic sanity check +:: Test compiling and running a native add-on +%node_gyp_exe% rebuild --directory=test\addons\hello-world --nodedir="%CD%" +if errorlevel 1 exit /b +Debug\node.exe test\addons\hello-world\test.js +if errorlevel 1 exit /b diff --git a/jenkins/scripts/windows/node-compile-windows.cmd b/jenkins/scripts/windows/node-compile-windows.cmd new file mode 100644 index 000000000..38288df5e --- /dev/null +++ b/jenkins/scripts/windows/node-compile-windows.cmd @@ -0,0 +1,47 @@ +:: Git +set GIT_REPO=%TEMP_REPO% +set GIT_REF=refs/heads/%TEMP_BRANCH% +call %~dp0git-checkout.cmd +if errorlevel 1 exit /b + +:: Delete binary branch that will be built here if it exists +git push %TEMP_REPO% :%TEMP_PUSH_BRANCH%-bin-%nodes% +:: reset errorlevel +ver > nul + +:: Compile +set "VCBUILD_EXTRA_ARGS=release msi" +call %~dp0compile.cmd +if errorlevel 1 exit /b + +:: Select files to pack +set "BINARY_FILES=config.gypi icu_config.gypi Release/node.exe Release/node.lib Release/openssl-cli.exe Release/cctest.exe Release/node.pdb" +if exist Release\node_test_engine.dll set "BINARY_FILES=%BINARY_FILES% Release/node_test_engine.dll" + +:: Setup binary package +rm -rf binary +mkdir binary +tar cavf binary/binary.tar.gz %BINARY_FILES% +if errorlevel 1 exit /b +md5sum binary/binary.tar.gz %BINARY_FILES% +:: make a copy for artifact +copy binary\binary.tar.gz . +if errorlevel 1 exit /b + +:: Create local temp branch with binary +git config --replace-all user.name "Node.js Jenkins CI" +git config --replace-all user.email ci@iojs.org +git checkout -B %TEMP_PUSH_BRANCH%-bin-%nodes% +if errorlevel 1 exit /b +git add binary\binary.tar.gz +if errorlevel 1 exit /b +git commit -m "added binaries" +if errorlevel 1 exit /b + +:: Push temp branch +git push "%TEMP_REPO%" "+%TEMP_PUSH_BRANCH%-bin-%nodes%" +if errorlevel 1 exit /b + +:: Delete local temp branch with binary +git checkout -f refs/remotes/jenkins_tmp +git branch -D %TEMP_PUSH_BRANCH%-bin-%nodes% diff --git a/jenkins/scripts/windows/node-test-binary-windows-js-suites.cmd b/jenkins/scripts/windows/node-test-binary-windows-js-suites.cmd new file mode 100644 index 000000000..3408b3443 --- /dev/null +++ b/jenkins/scripts/windows/node-test-binary-windows-js-suites.cmd @@ -0,0 +1,18 @@ +:: Git +set COMPILED_BY=%nodes:*-COMPILED_BY-=% +set GIT_REPO=%TEMP_REPO% +set GIT_REF=refs/heads/%TEMP_BRANCH%-bin-win-%COMPILED_BY% +call %~dp0git-checkout.cmd +if errorlevel 1 exit /b + +:: Run tests +set "test_ci_args=--run=%RUN_SUBSET%,4" +set VCBUILD_TARGET=test-ci-js +call %~dp0test.cmd +if errorlevel 1 exit /b + +:: Check existence of test results +if not exist js-tests.junit.xml ( + echo "Could not find test.tap or js-tests.junit.xml" + exit /b 1 +) diff --git a/jenkins/scripts/windows/node-test-binary-windows-native-suites.cmd b/jenkins/scripts/windows/node-test-binary-windows-native-suites.cmd new file mode 100644 index 000000000..e9c505ca8 --- /dev/null +++ b/jenkins/scripts/windows/node-test-binary-windows-native-suites.cmd @@ -0,0 +1,22 @@ +:: Git +set COMPILED_BY=%nodes:*-COMPILED_BY-=% +set GIT_REPO=%TEMP_REPO% +set GIT_REF=refs/heads/%TEMP_BRANCH%-bin-win-%COMPILED_BY% +call %~dp0git-checkout.cmd +if errorlevel 1 exit /b + +:: Run tests +set "test_ci_args=" +set VCBUILD_TARGET=test-ci-native +call %~dp0test.cmd +if errorlevel 1 exit /b + +:: Check existence of test results +if not exist js-tests.junit.xml ( + echo "Could not find test.tap or js-tests.junit.xml" + exit /b 1 +) +if not exist cctest.junit.xml ( + echo "Could not find cctest.tap or cctest.junit.xml" + exit /b 1 +) diff --git a/jenkins/scripts/windows/test.cmd b/jenkins/scripts/windows/test.cmd new file mode 100644 index 000000000..8af22daf0 --- /dev/null +++ b/jenkins/scripts/windows/test.cmd @@ -0,0 +1,65 @@ +git rev-parse --verify HEAD +git rev-parse "HEAD^" + +:: Extract binary +md5sum binary/binary.tar.gz +tar xzvf binary/binary.tar.gz +if errorlevel 1 exit /b +if exist out\Release\node.exe ln -s out/Release Release +md5sum config.gypi icu_config.gypi Release/node.exe Release/node.lib Release/openssl-cli.exe Release/cctest.exe + +:: Run the tests +call :diagnostics Before +set DEBUG_HELPER=1 +call vcbuild.bat release noprojgen nobuild ignore-flaky %VCBUILD_TARGET% +set "EXIT_RETURN_VALUE=%errorlevel%" +ver > nul +call :diagnostics After + +:: Convert test results +if exist test.tap ( + tap2junit -i test.tap -o js-tests.junit.xml + if errorlevel 1 exit /b +) +if exist cctest.tap ( + tap2junit -i cctest.tap -o cctest.junit.xml + if errorlevel 1 exit /b +) + +:: The JUnit Plugin only marks the job as Unstable when it finds any kind of failure, including flaky tests. +:: We need to use the return code of vcbuild.bat to fail the job when there are real failures. +exit /b %EXIT_RETURN_VALUE% + + + +:diagnostics +echo off +set "DIAGFILE=c:\jenkins_diagnostics.txt" +echo. >> %DIAGFILE% +echo. >> %DIAGFILE% +echo. >> %DIAGFILE% +set "TS=%date% %time%" +echo %TS% +echo %TS% >> %DIAGFILE% +echo %1 running vcbuild >> %DIAGFILE% +echo %BUILD_TAG% >> %DIAGFILE% +echo %BUILD_URL% >> %DIAGFILE% +echo %NODE_NAME% >> %DIAGFILE% +echo. >> %DIAGFILE% +echo netstat -abno >> %DIAGFILE% +netstat -abno >> %DIAGFILE% 2>&1 +echo. >> %DIAGFILE% +echo tasklist /v >> %DIAGFILE% +tasklist /v >> %DIAGFILE% 2>&1 +echo. >> %DIAGFILE% +echo tasklist /svc >> %DIAGFILE% +tasklist /svc >> %DIAGFILE% 2>&1 +echo. >> %DIAGFILE% +echo wmic path win32_process get Caption,Processid,Commandline >> %DIAGFILE% +wmic path win32_process get Caption,Processid,Commandline | more >> %DIAGFILE% 2>&1 +mv %DIAGFILE% %DIAGFILE%-OLD +cat %DIAGFILE%-OLD | sed s/\r//g;s/$/\r/ | tail -c 20000000 > %DIAGFILE% +rm %DIAGFILE%-OLD +echo on +tasklist | grep node +exit /b 0 From ef7aba91e1ba64f0e48c4347306c3400287d9a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Tue, 12 Nov 2019 15:17:59 +0000 Subject: [PATCH 03/17] ansible,win: disable SSLv2 and SSLv3 Following Azure security recommendations. --- .../roles/bootstrap/tasks/partials/win.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ansible/roles/bootstrap/tasks/partials/win.yml b/ansible/roles/bootstrap/tasks/partials/win.yml index 6ddb77fe1..5e3701837 100644 --- a/ansible/roles/bootstrap/tasks/partials/win.yml +++ b/ansible/roles/bootstrap/tasks/partials/win.yml @@ -26,3 +26,23 @@ name: AutoAdminLogon data: 1 type: string + +# Comply with Azure security recommendations +# After changing anything in this list check if the following still work: +# - Windows Update +# - PowerShell remoting (for Ansible connections) +# - RDP from Windows +# - RDP from Remmina +- block: + - name: disable SSL 2.0 + win_regedit: + path: 'HKLM:\System\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\SSL 2.0\Server' + name: Enabled + data: 0 + type: dword + - name: disable SSL 3.0 + win_regedit: + path: 'HKLM:\System\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\SSL 3.0\Server' + name: Enabled + data: 0 + type: dword From ec5b0c91ea4f28ad269a7163ae27e93bc6494ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Tue, 12 Nov 2019 15:20:12 +0000 Subject: [PATCH 04/17] ansible,win: don't install MFC and XP compat --- ansible/roles/visual-studio/tasks/partials/vs2015.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/visual-studio/tasks/partials/vs2015.yml b/ansible/roles/visual-studio/tasks/partials/vs2015.yml index 7a1159357..3324c2448 100644 --- a/ansible/roles/visual-studio/tasks/partials/vs2015.yml +++ b/ansible/roles/visual-studio/tasks/partials/vs2015.yml @@ -12,7 +12,7 @@ - name: install Visual Studio Community 2015 win_command: 'C:\TEMP\vs2015_community.exe /Silent /NoRestart - /InstallSelectableItems NativeLanguageSupport_Group + /InstallSelectableItems NativeLanguageSupport_VC /Log C:\TEMP\vs2015_install_log.txt' - name: install WiX Toolset From 2a756719c21dd2472d296c1807c8be8aa04b4b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Sun, 24 Nov 2019 03:03:31 +0000 Subject: [PATCH 05/17] ansible,win: add VS2019 support --- ansible/roles/visual-studio/tasks/main.yml | 36 +++++++++---------- .../visual-studio/tasks/partials/vs2017.yml | 2 +- .../visual-studio/tasks/partials/vs2019.yml | 28 +++++++++++++++ 3 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 ansible/roles/visual-studio/tasks/partials/vs2019.yml diff --git a/ansible/roles/visual-studio/tasks/main.yml b/ansible/roles/visual-studio/tasks/main.yml index 6ab7fe685..e160bd86e 100644 --- a/ansible/roles/visual-studio/tasks/main.yml +++ b/ansible/roles/visual-studio/tasks/main.yml @@ -4,26 +4,22 @@ # Install Visual Studio # -- when: vs == '2013' - block: - # TODO: Ensure no other versions are installed - - name: install Visual Studio 2013 - include_tasks: "partials/vs2013.yml" +- name: install Visual Studio 2013 + when: "'vs2013' in inventory_hostname" + include_tasks: "partials/vs2013.yml" -- when: vs == '2015' - block: - # TODO: Ensure no other versions are installed - - name: install Visual Studio 2015 - include_tasks: "partials/vs2015.yml" +- name: install Visual Studio 2015 + when: "'vs2015' in inventory_hostname" + include_tasks: "partials/vs2015.yml" -- when: vs == 'vcbt2015' - block: - # TODO: Ensure no other versions are installed - - name: install Visual C++ Build Tools 2015 - include_tasks: "partials/vcbt2015.yml" +- name: install Visual C++ Build Tools 2015 + when: "'vcbt2015' in inventory_hostname" + include_tasks: "partials/vcbt2015.yml" -- when: vs == '2017' - block: - # TODO: Ensure no other versions are installed - - name: install Visual Studio 2017 - include_tasks: "partials/vs2017.yml" +- name: install Visual Studio 2017 + when: "'vs2017' in inventory_hostname" + include_tasks: "partials/vs2017.yml" + +- name: install Visual Studio 2019 + when: "'vs2019' in inventory_hostname" + include_tasks: "partials/vs2019.yml" diff --git a/ansible/roles/visual-studio/tasks/partials/vs2017.yml b/ansible/roles/visual-studio/tasks/partials/vs2017.yml index b6423cdbb..69c6b39d5 100644 --- a/ansible/roles/visual-studio/tasks/partials/vs2017.yml +++ b/ansible/roles/visual-studio/tasks/partials/vs2017.yml @@ -21,7 +21,7 @@ - name: download WiX Toolset Visual Studio Extension win_get_url: - url: 'https://github.com/wixtoolset/VisualStudioExtension/releases/download/v0.9.21.62588/Votive2017.vsix' + url: 'https://github.com/wixtoolset/VisualStudioExtension/releases/download/v1.0.0.4/Votive2017.vsix' dest: 'C:\TEMP\Votive2017.vsix' - name: install WiX Toolset Visual Studio Extension diff --git a/ansible/roles/visual-studio/tasks/partials/vs2019.yml b/ansible/roles/visual-studio/tasks/partials/vs2019.yml new file mode 100644 index 000000000..575f5a744 --- /dev/null +++ b/ansible/roles/visual-studio/tasks/partials/vs2019.yml @@ -0,0 +1,28 @@ +--- + +# +# Install Visual Studio 2017 +# + + +- name: install Visual Studio Community 2019 + win_chocolatey: name=visualstudio2019community + +- name: install Visual Studio Community 2019 Native Desktop Workload + win_chocolatey: + name: visualstudio2019-workload-nativedesktop + params: '--add Microsoft.VisualStudio.Component.VC.Tools.ARM64 --add Microsoft.VisualStudio.Component.VC.ATL.ARM64' + +- name: install WiX Toolset + import_tasks: 'wixtoolset.yml' + +# Note: https://chocolatey.org/docs/helpers-install-chocolatey-vsix-package exists, but +# can't be used with VS2019 yet: https://github.com/chocolatey/choco/issues/1201 + +- name: download WiX Toolset Visual Studio Extension + win_get_url: + url: 'https://github.com/wixtoolset/VisualStudioExtension/releases/download/v1.0.0.4/Votive2019.vsix' + dest: 'C:\TEMP\Votive2019.vsix' + +- name: install WiX Toolset Visual Studio Extension + win_shell: '& "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\IDE\VSIXInstaller.exe" /quiet C:\TEMP\Votive2019.vsix' From 85aad05746723a1204425f2130e7a60dc0eed35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Sun, 24 Nov 2019 03:40:02 +0000 Subject: [PATCH 06/17] ansible,win: add clcache Specific commit and floating PRs known to work. --- .../roles/baselayout-windows/tasks/main.yml | 4 +++ ansible/roles/visual-studio/tasks/main.yml | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/ansible/roles/baselayout-windows/tasks/main.yml b/ansible/roles/baselayout-windows/tasks/main.yml index d678cc525..622f58330 100644 --- a/ansible/roles/baselayout-windows/tasks/main.yml +++ b/ansible/roles/baselayout-windows/tasks/main.yml @@ -17,6 +17,10 @@ params: '/GitAndUnixToolsOnPath' - name: enable long paths in Git win_command: 'git config --global core.longpaths true' + - name: set Git user name + win_command: 'git config --global user.name "{{inventory_hostname}}"' + - name: set Git user email + win_command: 'git config --global user.email "ci@iojs.org"' # Necessary for junit output in Jenkins jobs - name: install tap2junit diff --git a/ansible/roles/visual-studio/tasks/main.yml b/ansible/roles/visual-studio/tasks/main.yml index e160bd86e..2154bbcdd 100644 --- a/ansible/roles/visual-studio/tasks/main.yml +++ b/ansible/roles/visual-studio/tasks/main.yml @@ -23,3 +23,32 @@ - name: install Visual Studio 2019 when: "'vs2019' in inventory_hostname" include_tasks: "partials/vs2019.yml" + +# Install clcache and apply patches known to work +- block: + - name: check if clcache is already cloned + win_stat: path='C:\clcache' + register: clcache_stat + - name: clone clcache repository + win_command: 'git clone https://github.com/frerich/clcache.git C:\clcache' + when: not clcache_stat.stat.exists + - name: update clcache repository + win_command: 'git fetch --all' + args: { chdir: 'C:\clcache' } + when: clcache_stat.stat.exists + - name: checkout revision + win_command: 'git checkout -f -B master 7a3e62a3d801e0bc94dd78001c03144ce5232940' + args: { chdir: 'C:\clcache' } + - name: merge PR 324 + win_shell: 'curl.exe -L https://github.com/frerich/clcache/pull/324.patch | git am' + args: { chdir: 'C:\clcache' } + - name: merge PR 317 + win_shell: 'curl.exe -L https://github.com/frerich/clcache/pull/317.patch | git am' + args: { chdir: 'C:\clcache' } + - name: install PyInstaller + # https://github.com/pyinstaller/pyinstaller/issues/4265 + win_command: 'py -3 -m pip install https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz' + #win_command: 'py -3 -m pip install PyInstaller' + - name: build clcache + win_shell: '$env:PYTHONPATH = "C:\clcache"; py -3 -m PyInstaller -y pyinstaller/clcache_main.py' + args: { chdir: 'C:\clcache' } From cb15133fbb26fadf0dc061459efbda71a4183460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Sun, 24 Nov 2019 05:29:32 +0000 Subject: [PATCH 07/17] ansible,win: add Python 3 --- ansible/roles/baselayout-windows/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/roles/baselayout-windows/tasks/main.yml b/ansible/roles/baselayout-windows/tasks/main.yml index 622f58330..4dd958b10 100644 --- a/ansible/roles/baselayout-windows/tasks/main.yml +++ b/ansible/roles/baselayout-windows/tasks/main.yml @@ -7,6 +7,9 @@ - name: install NetWide Assembler win_chocolatey: name=nasm +- name: install Python 3 + win_chocolatey: name=python + - name: install Python 2 win_chocolatey: name=python2 @@ -24,7 +27,7 @@ # Necessary for junit output in Jenkins jobs - name: install tap2junit - win_command: 'pip2 install tap2junit' + win_command: 'pip3 install --upgrade tap2junit' # Necessary for the libuv test job - block: From de3451644689cabc6f47591fecb9fa54f1002524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Thu, 19 Dec 2019 16:59:35 +0000 Subject: [PATCH 08/17] ansible,win: add CMake Refs: https://github.com/nodejs/build/issues/2097 --- ansible/roles/baselayout-windows/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/roles/baselayout-windows/tasks/main.yml b/ansible/roles/baselayout-windows/tasks/main.yml index 4dd958b10..75dac30e1 100644 --- a/ansible/roles/baselayout-windows/tasks/main.yml +++ b/ansible/roles/baselayout-windows/tasks/main.yml @@ -7,6 +7,11 @@ - name: install NetWide Assembler win_chocolatey: name=nasm +- name: install CMake + win_chocolatey: + name: cmake + install_args: 'ADD_CMAKE_TO_PATH=System' + - name: install Python 3 win_chocolatey: name=python From 8659b6fad00b1815a33b33c13a23b0e7dab0cdc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Mon, 25 Nov 2019 19:20:08 +0000 Subject: [PATCH 09/17] ansible,win: add update-windows playbook --- ansible/playbooks/update-windows.yml | 80 ++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ansible/playbooks/update-windows.yml diff --git a/ansible/playbooks/update-windows.yml b/ansible/playbooks/update-windows.yml new file mode 100644 index 000000000..03f7bf5ba --- /dev/null +++ b/ansible/playbooks/update-windows.yml @@ -0,0 +1,80 @@ +--- + +# +# Updates Windows username, password and connection ports +# +# Usage: +# +# Set the following variables with the new values in the secret inventory file: +# - new_user - to change the username +# - new_password - to change the password +# - new_port - to change the WinRM connection port (default: 5986) +# - new_rdp_port - to change the RDP connection port (default: 3389) +# +# Changing username, password or WinRM port makes Ansible unable to connect, +# failing the command immediately. Thus, after EACH STEP in this script +# runs/fails successfully, remove the old variable and 'new_' from the new one +# in the inventory file and run again if there are more to change. +# +# Only the RDP port needs a reboot to apply (ansible HOST -m win_reboot). +# +# Changing credentials on release machines breaks access to the code signing +# certificate, so it need to be re-installed after running this. +# + + +- hosts: + - "*-win*" + + vars: + autologon_regpath: 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon' + + tasks: + - name: set automatic logon user name + when: '(new_user is defined) and (new_user|length > 0)' + win_regedit: + path: "{{ autologon_regpath }}" + name: DefaultUsername + data: "{{ new_user }}" + type: string + - name: rename user account - applies immediately making this fail on success + when: '(new_user is defined) and (new_user|length > 0)' + win_command: "wmic useraccount where name=\"{{ ansible_user }}\" rename {{ new_user }}" + + - name: set automatic logon password + when: '(new_password is defined) and (new_password|length > 0)' + win_regedit: + path: "{{ autologon_regpath }}" + name: DefaultPassword + data: "{{ new_password }}" + type: string + - name: change user password - applies immediately making this fail on success + when: '(new_password is defined) and (new_password|length > 0)' + win_command: "net user {{ ansible_user }} {{ new_password }}" + + +# CAUTION: Change ports only in Rackspace. Azure hosts are behind NAT. +- hosts: + - "*-rackspace-win*" + + vars: + netsh_common: 'netsh advfirewall firewall add rule profile=any dir=in protocol=TCP action=allow' + + tasks: + - name: add firewall exception for WinRM port + when: '(new_port is defined) and (new_port > 0)' + win_command: "{{ netsh_common }} name=\"Allow WinRM HTTPS on port {{ new_port }}\" localport={{ new_port }}" + - name: change WinRM port - applies immediately making this fail with ConnectTimeout on success + when: '(new_port is defined) and (new_port > 0)' + win_shell: "winrm set winrm/config/listener?Address=*+Transport=HTTPS '@{Port=\"{{ new_port }}\"}'" + + - name: add firewall exception for RDP port + when: '(new_rdp_port is defined) and (new_rdp_port > 0)' + win_command: "{{ netsh_common }} name=\"Allow RDP on port {{ new_rdp_port }}\" localport={{ new_rdp_port }}" + - name: change RDP port - applies only when host is rebooted + when: '(new_rdp_port is defined) and (new_rdp_port > 0)' + win_regedit: + path: 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' + name: PortNumber + data: "{{ new_rdp_port }}" + type: dword From ed98d5c2f7a5db94199cf3b486e991b0b1c3d7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Mon, 25 Nov 2019 19:18:46 +0000 Subject: [PATCH 10/17] ansible: accept 'password' shortcut --- ansible/ansible.cfg | 1 + ansible/plugins/inventory/nodejs_yaml.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 88ed4b9cd..ff02063f7 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -43,6 +43,7 @@ ansible_python_interpreter = /NODEJS2/python-2017-04-12-py27/python27/bin/python home = /u [hosts:win] +ansible_become = false ansible_connection = winrm ansible_winrm_server_cert_validation = ignore diff --git a/ansible/plugins/inventory/nodejs_yaml.py b/ansible/plugins/inventory/nodejs_yaml.py index 3a7fd0625..b528450aa 100755 --- a/ansible/plugins/inventory/nodejs_yaml.py +++ b/ansible/plugins/inventory/nodejs_yaml.py @@ -243,6 +243,10 @@ def parse_yaml(hosts, config): hostvars.update({'ansible_become': True}) del metadata['user'] + if 'password' in metadata: + hostvars.update({'ansible_password': str(metadata['password'])}) + del metadata['password'] + hostvars.update(metadata) # add specific options from config From ff0c5670a29b41043b6e3d0cc587b83ad734edca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Sun, 24 Nov 2019 03:11:14 +0000 Subject: [PATCH 11/17] ansible: move Windows to main create worker playbook --- .../jenkins/worker/create-windows.yml | 31 ------------------- ansible/playbooks/jenkins/worker/create.yml | 25 +++++++++++++++ .../tasks/partials/chocolatey.yml | 2 +- 3 files changed, 26 insertions(+), 32 deletions(-) delete mode 100644 ansible/playbooks/jenkins/worker/create-windows.yml diff --git a/ansible/playbooks/jenkins/worker/create-windows.yml b/ansible/playbooks/jenkins/worker/create-windows.yml deleted file mode 100644 index 7d4f89c72..000000000 --- a/ansible/playbooks/jenkins/worker/create-windows.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -# -# set up a jenkins worker -- muy bueno! -# - -- hosts: - - "*-win*" - - roles: - - bootstrap - - package-upgrade - - visual-studio - - baselayout-windows - - jenkins-worker-windows - - pre_tasks: - - name: check if secret is properly set - fail: - failed_when: not secret - - name: check if the Visual Studio version to install is properly set - when: os|startswith("win") - fail: - failed_when: not vs - - post_tasks: - - name: reboot Windows machines - when: os|startswith("win") - win_reboot: - - environment: '{{remote_env}}' diff --git a/ansible/playbooks/jenkins/worker/create.yml b/ansible/playbooks/jenkins/worker/create.yml index e91b9229b..245e56eb6 100644 --- a/ansible/playbooks/jenkins/worker/create.yml +++ b/ansible/playbooks/jenkins/worker/create.yml @@ -31,6 +31,28 @@ environment: '{{remote_env}}' +- hosts: + - "*-win*" + + roles: + - bootstrap + - package-upgrade + - baselayout-windows + - visual-studio + - jenkins-worker-windows + + pre_tasks: + - name: check if secret is properly set + fail: + failed_when: not secret + + post_tasks: + - name: reboot Windows machines + when: os|startswith("win") + win_reboot: + + environment: '{{remote_env}}' + # # Set up Jenkins Workspace servers # @@ -52,7 +74,10 @@ roles: - linux-perf +# # Ensure node is not installed anywhere but the linter servers +# + - hosts: - test - release diff --git a/ansible/roles/package-upgrade/tasks/partials/chocolatey.yml b/ansible/roles/package-upgrade/tasks/partials/chocolatey.yml index 583f2d60f..c34abb265 100644 --- a/ansible/roles/package-upgrade/tasks/partials/chocolatey.yml +++ b/ansible/roles/package-upgrade/tasks/partials/chocolatey.yml @@ -8,7 +8,7 @@ - name: reboot machine to ensure no updates are pending win_reboot: - name: wait for Jenkins to start if already installed - win_shell: 'Start-Sleep -Seconds 10' + win_shell: 'Start-Sleep -Seconds 30' - name: stop Jenkins after reboot win_shell: 'Get-WmiObject Win32_Process -Filter "(Name = ''cmd.exe'') AND (CommandLine LIKE ''%jenkins.bat%'')" | % {$_.Terminate()}' From 6f1a8f153cbbb6e1b55540de0cb6de805753daa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Wed, 4 Dec 2019 02:59:03 +0000 Subject: [PATCH 12/17] ansible: update Windows inventory --- ansible/inventory.yml | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 39e778e3c..f7b092e42 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -77,6 +77,20 @@ hosts: - azure: msft-ubuntu1604_arm_cross-x64-1: {ip: nodejs.eastus2.cloudapp.azure.com, user: ubuntu} + msft-win10_vcbt2015-x64-1: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win10_vcbt2015-x64-2: {ip: nodejs.westus2.cloudapp.azure.com} + msft-win10_vcbt2015-x64-3: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win10_vcbt2015-x64-4: {ip: nodejs.westus2.cloudapp.azure.com} + msft-win10_vs2019-x64-1: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win10_vs2019-x64-2: {ip: nodejs.westus2.cloudapp.azure.com} + msft-win10_vs2019-x64-3: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win10_vs2019-x64-4: {ip: nodejs.westus2.cloudapp.azure.com} + msft-win2016_vs2017-x64-1: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win2016_vs2017-x64-2: {ip: nodejs.westus2.cloudapp.azure.com} + msft-win2016_vs2017-x64-3: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win2016_vs2017-x64-4: {ip: nodejs.westus2.cloudapp.azure.com} + msft-win2016_vs2017-x64-5: {ip: nodejs.eastus2.cloudapp.azure.com} + msft-win2016_vs2017-x64-6: {ip: nodejs.westus2.cloudapp.azure.com} - digitalocean: debian8-x64-1: {ip: 159.203.103.52} @@ -134,8 +148,8 @@ hosts: macos10.12-x64-2: {ip: 207.254.58.162, port: 10002, user: administrator} - msft: - win10-arm64-1: { vs: '2017' } - win10-arm64-2: { vs: '2017' } + win10_vs2017-arm64-1: {} + win10_vs2017-arm64-2: {} - osuosl: @@ -159,6 +173,24 @@ hosts: fedora27-x64-1: {ip: 119.9.51.79} ubuntu1604-x64-1: {ip: 119.9.51.176} ubuntu1604-x64-2: {ip: 104.130.124.194} + win2008r2_vs2017-x64-1: {ip: 162.242.223.198} + win2008r2_vs2017-x64-2: {ip: 104.130.135.210} + win2008r2_vs2017-x64-3: {ip: 119.9.131.43} + win2008r2_vs2017-x64-4: {ip: 104.130.116.9} + win2012r2_vs2013-x64-1: {ip: 104.239.174.165} + win2012r2_vs2013-x64-2: {ip: 104.130.132.171} + win2012r2_vs2015-x64-1: {ip: 104.239.174.8} + win2012r2_vs2015-x64-2: {ip: 104.130.141.137} + win2012r2_vs2017-x64-1: {ip: 162.242.237.72} + win2012r2_vs2017-x64-2: {ip: 104.239.142.99} + win2012r2_vs2017-x64-3: {ip: 119.9.131.54} + win2012r2_vs2017-x64-4: {ip: 166.78.99.25} + win2012r2_vs2019-x64-1: {ip: 162.242.237.124} + win2012r2_vs2019-x64-2: {ip: 104.130.158.58} + win2012r2_vs2019-x64-3: {ip: 119.9.131.63} + win2012r2_vs2019-x64-4: {ip: 104.130.219.103} + win2012r2_vs2019-x64-5: {ip: 104.130.6.92} + win2012r2_vs2019-x64-6: {ip: 104.130.141.231} - requireio: andineck-debian10-armv6l_pi1p-1: {ip: 192.168.2.42, user: pi, alias: iojs-ns-pi1p-3 } From a9da63f404a3e6b569735d9a1b32c602ffd9d69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Thu, 12 Sep 2019 10:39:36 +0100 Subject: [PATCH 13/17] doc,win: update non-ansible-configuration-notes.md --- doc/non-ansible-configuration-notes.md | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/doc/non-ansible-configuration-notes.md b/doc/non-ansible-configuration-notes.md index a1a0fc3e0..cc02b4056 100644 --- a/doc/non-ansible-configuration-notes.md +++ b/doc/non-ansible-configuration-notes.md @@ -25,32 +25,20 @@ Machines should have: - Remote Desktop (RDP) enabled, the port should be listed with the access credentials if it is not the default (3389). - PowerShell access enabled, the port should be listed with the access credentials if it is not the default (5986). -To use Ansible for Windows, PowerShell access should be enabled as described in [`ansible.intro_windows`][]. - ### Control machine (where Ansible is run) Install the `pywinrm` pip module: `pip install pywinrm` ### Target machines -Ensure PowerShell v3 or higher is installed (`$PSVersionTable.PSVersion`), refer to [`ansible.intro_windows`][] if not. - -Before running the preparation script, the network location must be set to Private (not necessary for Azure). -This can be done in Windows 10 by going to `Settings`, `Network`, `Ethernet`, click the connection name -(usually `Ethernet`, next to the icon) and change `Find devices and content` to on. - -The preparation script can be manually downloaded from [`ansible.intro_windows`][] and run, or automatically by running -this in PowerShell (run as Administrator): +The preparation script needs to be run in PowerShell (run as Administrator): ```powershell -Set-ExecutionPolicy -Force -Scope CurrentUser Unrestricted -$ansibleURL = "https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1" -Invoke-WebRequest $ansibleURL -OutFile ConfigureRemotingForAnsible.ps1 -.\ConfigureRemotingForAnsible.ps1 -# Optional -rm ConfigureRemotingForAnsible.ps1 +iwr -useb https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1 | iex ``` +Test the connection to the target machine with `ansible HOST -m win_ping -vvvv`. If there is any issue, please refer to the official Ansible documentation in [Setting up a Windows Host][]. + ## macOS TODO: Update and copy notes from @@ -305,6 +293,6 @@ yum install -y devtoolset-6-s390x-rpms/* -[`ansible.intro_windows`]: http://docs.ansible.com/ansible/intro_windows.html +[Setting up a Windows Host]: https://docs.ansible.com/ansible/latest/user_guide/windows_setup.html [newer Ansible configuration]: https://github.com/nodejs/build/tree/master/ansible [stand-alone]: https://github.com/nodejs/build/tree/master/setup/windows From 7144fc50b4aa070eebf407572cc436bab140c637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Tue, 22 Oct 2019 03:59:09 +0100 Subject: [PATCH 14/17] doc,win: update Windows supported versions --- ...windows-visualstudio-supported-versions.md | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/windows-visualstudio-supported-versions.md b/doc/windows-visualstudio-supported-versions.md index 65f088856..5dfa1dcec 100644 --- a/doc/windows-visualstudio-supported-versions.md +++ b/doc/windows-visualstudio-supported-versions.md @@ -18,6 +18,9 @@ Supported versions for running the Node.js installer and executable as released. | v9 | 7 / 2008 R2 | | v10 | 7 / 2008 R2 | | v11 | 7 / 2008 R2 | +| v12 | 7 / 2008 R2 | +| v13 | 7 / 2008 R2 | +| v14 | 8.1 / 2012 R2 | ## For building Node.js Core @@ -34,6 +37,9 @@ Supported versions for building Node.js from source. | v9 | 2015, VCBT2015, 2017 | | v10 | 2017 [5] | | v11 | 2017 | +| v12 | 2017, 2019 (flag) [8] | +| v13 | 2017, 2019 [9] | +| v14 | 2017, 2019 | ## For building Node.js Addons @@ -49,6 +55,9 @@ Supported versions for building Node.js addons. End-users should have one of the | v9 | 2015, VCBT2015, 2017 [7] | | v10 | 2015, VCBT2015, 2017 | | v11 | 2015, VCBT2015, 2017 | +| v12 | 2015, VCBT2015, 2017, 2019 [10] | +| v13 | 2015, VCBT2015, 2017, 2019 | +| v14 | 2015, VCBT2015, 2017, 2019 | ## Official Releases @@ -65,6 +74,9 @@ These versions are used to build the official releases. | v9 | 2008 R2 | 2015 | | v10 | 2012 R2 | 2017 [5] | | v11 | 2012 R2 | 2017 | +| v12 | 2012 R2 | 2017 | +| v13 | 2012 R2 | 2017 | +| v14 | 2012 R2 | 2019 | ## References @@ -82,8 +94,16 @@ These versions are used to build the official releases. 5. Support for Visual Studio 2015 was removed in v10.0.0. - Pull Request: https://github.com/nodejs/node/pull/16868 - Pull Request: https://github.com/nodejs/node/pull/16969 -6. Support for **building addons** with Visual Studio 2017 was added in v8.0.0. +6. Support for **building addons** with Visual Studio 2017 was added in v8.0.0 (node-gyp v3.6.0). - Pull Request: https://github.com/nodejs/node-gyp/pull/1130 + - Pull Request: https://github.com/nodejs/node/pull/12480 7. Support for **building addons** with Visual Studio 2013 was removed in v9.0.0. - Issue: https://github.com/nodejs/node/issues/13372 - Pull Request: https://github.com/nodejs/node/pull/14764 +8. Support for Visual Studio 2019 was added behind a flag in v12.8.0. + - Pull Request: https://github.com/nodejs/node/pull/28781 +9. Support for Visual Studio 2019 by default was added in v13.0.1. + - Pull Request: https://github.com/nodejs/node/pull/30022 +10. Support for **building addons** with Visual Studio 2019 was added in v12.8.0 (node-gyp 5.0.0). + - Pull Request: https://github.com/nodejs/node-gyp/pull/1762 + - Pull Request: https://github.com/nodejs/node/pull/28853 From 88c5ec6349968eebce98193345765158e8311c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Tue, 19 Nov 2019 12:17:35 +0000 Subject: [PATCH 15/17] doc,win: update node-test-commit-matrix.md --- doc/node-test-commit-matrix.md | 135 +++++++++++++++++++++++---------- 1 file changed, 94 insertions(+), 41 deletions(-) diff --git a/doc/node-test-commit-matrix.md b/doc/node-test-commit-matrix.md index 0625c929b..08f723276 100644 --- a/doc/node-test-commit-matrix.md +++ b/doc/node-test-commit-matrix.md @@ -42,48 +42,12 @@ This is assumed correct as of the date of last commit. If you notice a discrepan - smartos16-64 (Node >= 8 < 12) - smartos17-64 (Node >= 10 < 12) - smartos18-64 (Node >= 12) - - **node-test-commit-windows-fanned** + - **node-test-commit-windows-fanned** + [See below for detailed information](#windows-test-matrix). Sub-jobs are: - **node-compile-windows** - * Combination Filter (structured as a conjunction of implications): - - When `ENGINE!="ChakraCore"` exclude `*-arm` - - When Node >= 6 exclude `win-vs2013*` - - When Node >= 10 exclude `win-vs2015*` and `win-vcbt2015*` - - When Node < 8 exclude `win-vs2017*` - - When Node < 10 exclude `win-vs2017-x86` - - win-vcbt2015 (Node < 10) - - win-vs2015 (Node >= 6 < 10) - - win-vs2015-arm (Node-ChakraCore) - - win-vs2015-x86 (Node >= 6 < 10) - - win-vs2017 (Node >= 8) - - win-vs2017-arm (Node-ChakraCore) - - win-vs2017-x86 (Node >= 10) - - **node-test-binary-windows** (Node <= 10) - * Combination Filter (structured as a conjunction of implications): - - When Node < 10: - - On `win10` run: - - `COMPILED_BY=="vcbt2015"` always - - On `win2008r2-vs2013` run: - - `COMPILED_BY=="vs2013"` when Node < 6 - - `COMPILED_BY=="vs2015"` when Node >= 6 < 9 - - On `win2008r2-vs2017` run: - - `COMPILED_BY=="vs2015"` when Node >= 9 - - On `win2012r2` run: - - `COMPILED_BY=="vs2015"` always - - `COMPILED_BY=="vs2015-x86"` always - - On `win2016` run: - - `COMPILED_BY=="vs2015"` when Node < 8 - - `COMPILED_BY=="vs2017"` when Node >= 8 - - When Node == 10: - - On `win10` run `COMPILED_BY=="vs2017"` - - On `win2008r2-vs2017` run `COMPILED_BY=="vs2017"` - - On `win2012r2` run `COMPILED_BY=="vs2017-x86"` - - On `win2016` run `COMPILED_BY=="vs2017"` - - **node-test-binary-windows-2** (Node >= 11) - * Combination Filter: - - On `win10` run `COMPILED_BY=="vs2017"` - - On `win2008r2-vs2017` run `COMPILED_BY=="vs2017"` - - On `win2012r2` run `COMPILED_BY=="vs2017-x86"` - - On `win2016` run `COMPILED_BY=="vs2017"` + - **node-test-commit-windows-debug** + - **node-test-binary-windows-js-suites** + - **node-test-binary-windows-native-suites** - **node-test-commit-linux-containered** - ubuntu1604_sharedlibs_debug_x64 1. `CONFIG_FLAGS="$CONFIG_FLAGS --debug" make build-ci -j $JOBS` @@ -137,3 +101,92 @@ This is assumed correct as of the date of last commit. If you notice a discrepan 1. `make -j1 bench-addons-build` 1. `make -j 4 build-ci` 1. `python tools/test.py -j 4 -p tap --logfile test.tap --mode=release --flaky-tests=dontcare --worker` + +## Windows Test Matrix + +The **node-test-commit-windows-fanned** is divided in 3 stages: + - **git-rebase** loads a commit from GitHub into a repository dedicated to temporary data, optionally rebasing. + - **node-compile-windows** compiles Node.js, loading from and storing the result back in the temporaries repository. Meanwhile, **node-compile-windows-debug** compiles Node.js in debug mode and compiles a native add-on to ensure minimum functionality, discarding the resulting binaries. + - **node-test-binary-windows-js-suites** and **node-test-binary-windows-native-suites** fully test the binaries from the previous stage (only from the Release builds). + +Running the resulting binaries from every supported Visual Studio version in every supported Windows version would place an unreasonable demand on the infrastructure we have available, so **only a subset of combinations is run**. +- Debug builds run only to ensure the debug build is not broken, the resulting binaries are not fully tested. +- x86 builds only run in one Visual Studio version, preferably the one used to build the official releases. + +### Combinations being tested + +- Run JS tests on all supported Windows Versions. +- Run add-ons tests on all supported Visual Studio versions (preferably spanning all supported Windows Versions). +- If necessary, run more combinations for the Visual Studio version that is used to build the official releases. + +The following combinations are used: + +#### Node.js version 8 + +| Binaries produced by: | Run in Windows version: | Test add-ons with: | +|-----------------------|-------------------------|--------------------| +| VCBT 2015 | 10 | VCBT 2015 | +| VS 2015 | 2008R2 | VS 2017 | +| | 2012R2 | VS 2013, VS 2015 | +| VS 2015 (x86) | 2012R2 | VS 2015 | +| VS 2017 | 2016 | VS 2017 | + +#### Node.js versions 9 to 12 + +| Binaries produced by: | Run in Windows version: | Test add-ons with: | +|-----------------------|-------------------------|--------------------| +| VS 2017 | 10 | VCBT 2015 | +| | 2008R2 (until EOL) | VS 2017 | +| | 2016 | VS 2017 | +| VS 2017 (x86) | 2012R2 | VS 2015 | + +#### Node.js versions 13 and newer + +| Binaries produced by: | Run in Windows version: | Test add-ons with: | +|-----------------------|-------------------------|--------------------| +| VS 2017 | 2016 | VS 2017 | +| | 2008R2 (until EOL) | VS 2017 | +| VS 2019 | 10 | VCBT 2015, VS 2019 | +| VS 2019 (x86) | 2012R2 | VS 2015, VS 2019 | + +### Machines available to CI + +| # | Windows | Visual Studio | Use for | Provider | Notes | +|---|---------|---------------|-----------|-----------|-------| +| 4 | 10 | VCBT 2015 | All | Azure | | +| 4 | 10 | VS 2019 | Test Only | Azure | | +| 6 | 2016 | VS 2017 | Test Only | Azure | | +| 4 | 2008R2 | VS 2017 | Test Only | Rackspace | Remove at Windows 2008 EOL | +| 2 | 2012R2 | VS 2013 | All | Rackspace | Remove at Node.js 8 EOL | +| 2 | 2012R2 | VS 2015 | All | Rackspace | | +| 4 | 2012R2 | VS 2017 | All | Rackspace | | +| 6 | 2012R2 | VS 2019 | All | Rackspace | | + +- Machines for "Test Only" don't build Node.js, but are still used to build add-ons so Visual Studio is necessary. +- Machines in Rackspace have more capacity, so avoid building on Azure, at least the current Node.js version. +- Optimize resources for the current Node.js master. +- Ensure Node.js LTS versions have full coverage, even if with possible bottlenecks in certain versions. + +### Jenkins Labels + +Machines for "Test Only" should have the following labels: + +| Template | Examples | Notes | +|---|---|---| +| `winVER` | `win10` | Used for other CI jobs that don't require VS | +| `winVER-vsVER` | `win10-vcbt2015`, `win2012r2-vs2019` | Used for other CI jobs | +| 1 | `win10-COMPILED_BY-vcbt2015` | Used for running JS tests | +| 2 | `win10-vs2015-COMPILED_BY-vs2017` | Used for running add-ons tests | + +All combinations of labels possible from the tables above (Combinations being tested) need to be assigned to the machines: +1. `Run in Windows version`-COMPILED_BY-`Binaries produced by` +2. `Run in Windows version`-`Test add-ons with`-COMPILED_BY-`Binaries produced by` + +- When building add-ons with a x86 Node.js version, the addons will automatically be compiled for x86 so a specific label (`winV-vsV-x86-COMPILED_BY-vsV`) is not needed. + +Machines for "All" ("Test Only" + Build) should also have the labels: + +| Template | Examples | Notes | +|---|---|---| +| `win-vsVER` | `win-vs2019` | | +| `win-vsVER-x86` | `win-vs2019-x86` | Not necessary for `vcbt2015` | From a5edf2472f2f4446d577d7ea6ee250604e86f524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Fri, 20 Dec 2019 02:12:25 +0000 Subject: [PATCH 16/17] fixup! doc,win: update node-test-commit-matrix.md --- doc/node-test-commit-matrix.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/doc/node-test-commit-matrix.md b/doc/node-test-commit-matrix.md index 08f723276..2b98d08e1 100644 --- a/doc/node-test-commit-matrix.md +++ b/doc/node-test-commit-matrix.md @@ -121,17 +121,7 @@ Running the resulting binaries from every supported Visual Studio version in eve The following combinations are used: -#### Node.js version 8 - -| Binaries produced by: | Run in Windows version: | Test add-ons with: | -|-----------------------|-------------------------|--------------------| -| VCBT 2015 | 10 | VCBT 2015 | -| VS 2015 | 2008R2 | VS 2017 | -| | 2012R2 | VS 2013, VS 2015 | -| VS 2015 (x86) | 2012R2 | VS 2015 | -| VS 2017 | 2016 | VS 2017 | - -#### Node.js versions 9 to 12 +#### Node.js versions 10 & 12 | Binaries produced by: | Run in Windows version: | Test add-ons with: | |-----------------------|-------------------------|--------------------| From a88ba93b432cb5c2b1e47ea60e4257014fc82c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reis?= Date: Sun, 29 Dec 2019 03:10:49 +0000 Subject: [PATCH 17/17] fixup! ansible,win: add VS2019 support --- ansible/roles/visual-studio/tasks/partials/vs2019.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/visual-studio/tasks/partials/vs2019.yml b/ansible/roles/visual-studio/tasks/partials/vs2019.yml index 575f5a744..cb618c1ca 100644 --- a/ansible/roles/visual-studio/tasks/partials/vs2019.yml +++ b/ansible/roles/visual-studio/tasks/partials/vs2019.yml @@ -1,7 +1,7 @@ --- # -# Install Visual Studio 2017 +# Install Visual Studio 2019 #