From e3e977fd8470cf359f15de00c19bfd0d77b65854 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 18 Nov 2021 15:25:33 -0500 Subject: [PATCH] Support node.js 16 and bump node.js 12 version. (#1439) * Support node.js 16 and bump node.js 12 version. * L0 --- src/Misc/externals.sh | 12 +- src/Runner.Worker/ActionManager.cs | 2 + src/Runner.Worker/ActionManifestManager.cs | 6 +- .../Handlers/NodeScriptActionHandler.cs | 4 +- src/Runner.Worker/Handlers/StepHost.cs | 12 +- src/Runner.Worker/action_yaml.json | 10 +- src/Test/L0/Worker/ActionManagerL0.cs | 76 +++++++++++- src/Test/L0/Worker/ActionManifestManagerL0.cs | 44 +++++++ src/Test/L0/Worker/StepHostL0.cs | 109 ++++++++++++++++++ src/Test/TestData/node16action.yml | 20 ++++ 10 files changed, 275 insertions(+), 20 deletions(-) create mode 100644 src/Test/L0/Worker/StepHostL0.cs create mode 100644 src/Test/TestData/node16action.yml diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index bc90fea1c63..fe7a74b9a0d 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -3,7 +3,8 @@ PACKAGERUNTIME=$1 PRECACHE=$2 NODE_URL=https://nodejs.org/dist -NODE12_VERSION="12.13.1" +NODE12_VERSION="12.22.7" +NODE16_VERSION="16.13.0" get_abs_path() { # exploits the fact that pwd will print abs path when no args @@ -126,6 +127,8 @@ function acquireExternalTool() { if [[ "$PACKAGERUNTIME" == "win-x64" || "$PACKAGERUNTIME" == "win-x86" ]]; then acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/$PACKAGERUNTIME/node.exe" node12/bin acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/$PACKAGERUNTIME/node.lib" node12/bin + acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin + acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin if [[ "$PRECACHE" != "" ]]; then acquireExternalTool "https://github.com/microsoft/vswhere/releases/download/2.6.7/vswhere.exe" vswhere fi @@ -134,18 +137,23 @@ fi # Download the external tools only for OSX. if [[ "$PACKAGERUNTIME" == "osx-x64" ]]; then acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-darwin-x64.tar.gz" node12 fix_nested_dir + acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-x64.tar.gz" node16 fix_nested_dir fi # Download the external tools for Linux PACKAGERUNTIMEs. if [[ "$PACKAGERUNTIME" == "linux-x64" ]]; then acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-linux-x64.tar.gz" node12 fix_nested_dir - acquireExternalTool "https://vstsagenttools.blob.core.windows.net/tools/nodejs/${NODE12_VERSION}/alpine/x64/node-${NODE12_VERSION}-alpine-x64.tar.gz" node12_alpine + acquireExternalTool "https://vstsagenttools.blob.core.windows.net/tools/nodejs/${NODE12_VERSION}/alpine/x64/node-v${NODE12_VERSION}-alpine-x64.tar.gz" node12_alpine + acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-x64.tar.gz" node16 fix_nested_dir + acquireExternalTool "https://vstsagenttools.blob.core.windows.net/tools/nodejs/${NODE16_VERSION}/alpine/x64/node-v${NODE16_VERSION}-alpine-x64.tar.gz" node16_alpine fi if [[ "$PACKAGERUNTIME" == "linux-arm64" ]]; then acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-linux-arm64.tar.gz" node12 fix_nested_dir + acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-arm64.tar.gz" node16 fix_nested_dir fi if [[ "$PACKAGERUNTIME" == "linux-arm" ]]; then acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-linux-armv7l.tar.gz" node12 fix_nested_dir + acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-armv7l.tar.gz" node16 fix_nested_dir fi diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 4cebea81b96..76b087a4b71 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -1199,6 +1199,8 @@ public sealed class NodeJSActionExecutionData : ActionExecutionData public string Pre { get; set; } public string Post { get; set; } + + public string NodeVersion { get; set; } } public sealed class PluginActionExecutionData : ActionExecutionData diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 741c39f5b24..3740128443f 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -451,7 +451,8 @@ private ActionExecutionData ConvertRuns( }; } } - else if (string.Equals(usingToken.Value, "node12", StringComparison.OrdinalIgnoreCase)) + else if (string.Equals(usingToken.Value, "node12", StringComparison.OrdinalIgnoreCase)|| + string.Equals(usingToken.Value, "node16", StringComparison.OrdinalIgnoreCase)) { if (string.IsNullOrEmpty(mainToken?.Value)) { @@ -461,6 +462,7 @@ private ActionExecutionData ConvertRuns( { return new NodeJSActionExecutionData() { + NodeVersion = usingToken.Value, Script = mainToken.Value, Pre = preToken?.Value, InitCondition = preIfToken?.Value ?? "always()", @@ -490,7 +492,7 @@ private ActionExecutionData ConvertRuns( } else { - throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead."); + throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker', 'node12' or 'node16' instead."); } } else if (pluginToken != null) diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index d4aa920c599..9ea6d12d8d4 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -83,7 +83,7 @@ public async Task RunAsync(ActionRunStage stage) HasPreStep = Data.HasPre, HasPostStep = Data.HasPost, IsEmbedded = ExecutionContext.IsEmbedded, - Type = "node12" + Type = Data.NodeVersion }; ExecutionContext.Root.ActionsStepsTelemetry.Add(telemetry); } @@ -99,7 +99,7 @@ public async Task RunAsync(ActionRunStage stage) workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); } - var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext); + var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion); string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}"); // Format the arguments passed to node. diff --git a/src/Runner.Worker/Handlers/StepHost.cs b/src/Runner.Worker/Handlers/StepHost.cs index 0907eaed23e..8741c22bbe5 100644 --- a/src/Runner.Worker/Handlers/StepHost.cs +++ b/src/Runner.Worker/Handlers/StepHost.cs @@ -23,7 +23,7 @@ public interface IStepHost : IRunnerService string ResolvePathForStepHost(string path); - Task DetermineNodeRuntimeVersion(IExecutionContext executionContext); + Task DetermineNodeRuntimeVersion(IExecutionContext executionContext, string preferredVersion); Task ExecuteAsync(string workingDirectory, string fileName, @@ -58,9 +58,9 @@ public string ResolvePathForStepHost(string path) return path; } - public Task DetermineNodeRuntimeVersion(IExecutionContext executionContext) + public Task DetermineNodeRuntimeVersion(IExecutionContext executionContext, string preferredVersion) { - return Task.FromResult("node12"); + return Task.FromResult(preferredVersion); } public async Task ExecuteAsync(string workingDirectory, @@ -123,7 +123,7 @@ public string ResolvePathForStepHost(string path) } } - public async Task DetermineNodeRuntimeVersion(IExecutionContext executionContext) + public async Task DetermineNodeRuntimeVersion(IExecutionContext executionContext, string preferredVersion) { // Best effort to determine a compatible node runtime // There may be more variation in which libraries are linked than just musl/glibc, @@ -148,14 +148,14 @@ public async Task DetermineNodeRuntimeVersion(IExecutionContext executio var msg = $"JavaScript Actions in Alpine containers are only supported on x64 Linux runners. Detected {os} {arch}"; throw new NotSupportedException(msg); } - nodeExternal = "node12_alpine"; + nodeExternal = $"{preferredVersion}_alpine"; executionContext.Debug($"Container distribution is alpine. Running JavaScript Action with external tool: {nodeExternal}"); return nodeExternal; } } } // Optimistically use the default - nodeExternal = "node12"; + nodeExternal = preferredVersion; executionContext.Debug($"Running JavaScript Action with default external tool: {nodeExternal}"); return nodeExternal; } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 3d766b851b0..b01d913482f 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -46,7 +46,7 @@ "runs": { "one-of": [ "container-runs", - "node12-runs", + "node-runs", "plugin-runs", "composite-runs" ] @@ -80,7 +80,7 @@ "loose-value-type": "string" } }, - "node12-runs": { + "node-runs": { "mapping": { "properties": { "using": "non-empty-string", @@ -112,10 +112,10 @@ "item-type": "composite-step" } }, - "composite-step":{ + "composite-step": { "one-of": [ "run-step", - "uses-step" + "uses-step" ] }, "run-step": { @@ -254,4 +254,4 @@ } } } -} +} \ No newline at end of file diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 62701bde859..5334b236f2a 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -965,7 +965,7 @@ public async void PrepareActions_CompositeActionWithActionfile_CompositePrestepN }; //Act - var result = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var result = await _actionManager.PrepareActionsAsync(_ec.Object, actions); //Assert Assert.Equal(1, result.PreStepTracker.Count); @@ -1288,7 +1288,7 @@ public void LoadsContainerActionDefinitionRegistry() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsNodeActionDefinition() + public void LoadsNode12ActionDefinition() { try { @@ -1344,8 +1344,78 @@ public void LoadsNodeActionDefinition() Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); Assert.NotNull(definition.Data.Execution); // execution - Assert.NotNull((definition.Data.Execution as NodeJSActionExecutionData)); + Assert.NotNull(definition.Data.Execution as NodeJSActionExecutionData); + Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); + Assert.Equal("node12", (definition.Data.Execution as NodeJSActionExecutionData).NodeVersion); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsNode16ActionDefinition() + { + try + { + // Arrange. + Setup(); + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node16' + main: 'task.js' +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull(definition.Data.Execution as NodeJSActionExecutionData); Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); + Assert.Equal("node16", (definition.Data.Execution as NodeJSActionExecutionData).NodeVersion); } finally { diff --git a/src/Test/L0/Worker/ActionManifestManagerL0.cs b/src/Test/L0/Worker/ActionManifestManagerL0.cs index cd03b8bbd8d..d11f3c1e85c 100644 --- a/src/Test/L0/Worker/ActionManifestManagerL0.cs +++ b/src/Test/L0/Worker/ActionManifestManagerL0.cs @@ -408,6 +408,50 @@ public void Load_NodeAction() var nodeAction = result.Execution as NodeJSActionExecutionData; Assert.Equal("main.js", nodeAction.Script); + Assert.Equal("node12", nodeAction.NodeVersion); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Load_Node16Action() + { + try + { + //Arrange + Setup(); + + var actionManifest = new ActionManifestManager(); + actionManifest.Initialize(_hc); + + //Act + var result = actionManifest.Load(_ec.Object, Path.Combine(TestUtil.GetTestDataPath(), "node16action.yml")); + + //Assert + Assert.Equal("Hello World", result.Name); + Assert.Equal("Greet the world and record the time", result.Description); + Assert.Equal(2, result.Inputs.Count); + Assert.Equal("greeting", result.Inputs[0].Key.AssertString("key").Value); + Assert.Equal("Hello", result.Inputs[0].Value.AssertString("value").Value); + Assert.Equal("entryPoint", result.Inputs[1].Key.AssertString("key").Value); + Assert.Equal("", result.Inputs[1].Value.AssertString("value").Value); + Assert.Equal(1, result.Deprecated.Count); + + Assert.True(result.Deprecated.ContainsKey("greeting")); + result.Deprecated.TryGetValue("greeting", out string value); + Assert.Equal("This property has been deprecated", value); + + Assert.Equal(ActionExecutionType.NodeJS, result.Execution.ExecutionType); + + var nodeAction = result.Execution as NodeJSActionExecutionData; + + Assert.Equal("main.js", nodeAction.Script); + Assert.Equal("node16", nodeAction.NodeVersion); } finally { diff --git a/src/Test/L0/Worker/StepHostL0.cs b/src/Test/L0/Worker/StepHostL0.cs new file mode 100644 index 00000000000..8d0a102efc6 --- /dev/null +++ b/src/Test/L0/Worker/StepHostL0.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Moq; +using Xunit; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Handlers; +using GitHub.Runner.Worker.Container; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class StepHostL0 + { + private Mock _ec; + private Mock _dc; + private TestHostContext CreateTestContext([CallerMemberName] String testName = "") + { + var hc = new TestHostContext(this, testName); + + _ec = new Mock(); + _ec.SetupAllProperties(); + _ec.Setup(x => x.Global).Returns(new GlobalContext { WriteDebug = true }); + var trace = hc.GetTrace(); + _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); }); + + _dc = new Mock(); + hc.SetSingleton(_dc.Object); + return hc; + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task DetermineNodeRuntimeVersionInContainerAsync() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange. + var sh = new ContainerStepHost(); + sh.Initialize(hc); + sh.Container = new ContainerInfo() { ContainerId = "1234abcd" }; + + _dc.Setup(d => d.DockerExec(_ec.Object, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(0); + + // Act. + var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node12"); + + // Assert. + Assert.Equal("node12", nodeVersion); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task DetermineNodeRuntimeVersionInAlpineContainerAsync() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange. + var sh = new ContainerStepHost(); + sh.Initialize(hc); + sh.Container = new ContainerInfo() { ContainerId = "1234abcd" }; + + _dc.Setup(d => d.DockerExec(_ec.Object, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .Callback((IExecutionContext ec, string id, string options, string command, List output) => + { + output.Add("alpine"); + }) + .ReturnsAsync(0); + + // Act. + var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node16"); + + // Assert. + Assert.Equal("node16_alpine", nodeVersion); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task DetermineNodeRuntimeVersionInUnknowContainerAsync() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange. + var sh = new ContainerStepHost(); + sh.Initialize(hc); + sh.Container = new ContainerInfo() { ContainerId = "1234abcd" }; + + _dc.Setup(d => d.DockerExec(_ec.Object, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .Callback((IExecutionContext ec, string id, string options, string command, List output) => + { + output.Add("github"); + }) + .ReturnsAsync(0); + + // Act. + var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node16"); + + // Assert. + Assert.Equal("node16", nodeVersion); + } + } + } +} diff --git a/src/Test/TestData/node16action.yml b/src/Test/TestData/node16action.yml new file mode 100644 index 00000000000..ca773bf98c1 --- /dev/null +++ b/src/Test/TestData/node16action.yml @@ -0,0 +1,20 @@ +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'Test Corporation' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + deprecationMessage: 'This property has been deprecated' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node16' + main: 'main.js' \ No newline at end of file