From a3bac7957615b878655de2ccb813e791308ede31 Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Thu, 19 Sep 2024 00:41:45 +0000 Subject: [PATCH 1/7] Adding Snapshot additional mapping tokens --- src/Runner.Worker/JobExtension.cs | 2 +- .../PipelineTemplateConstants.cs | 1 + .../PipelineTemplateConverter.cs | 33 ++++++++++++- src/Sdk/DTPipelines/Pipelines/Snapshot.cs | 11 +++++ src/Sdk/DTPipelines/workflow-v1.0.json | 19 ++++++- src/Test/L0/Worker/JobExtensionL0.cs | 49 ++++++++++++++++--- 6 files changed, 106 insertions(+), 9 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index a36e4beb4d8..e111a77aac5 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -403,7 +403,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel var snapshotOperationProvider = HostContext.GetService(); jobContext.RegisterPostJobStep(new JobExtensionRunner( runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest), - condition: $"{PipelineTemplateConstants.Success}()", + condition: snapshotRequest.Condition, displayName: $"Create custom image", data: null)); } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index a7e90fce334..8d81c7d2d53 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -30,6 +30,7 @@ public sealed class PipelineTemplateConstants public const String If = "if"; public const String Image = "image"; public const String ImageName = "image-name"; + public const String CustomImageVersion = "version"; public const String Include = "include"; public const String Inputs = "inputs"; public const String Job = "job"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 9d2c0bdca7a..9105dc40a33 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -349,6 +349,10 @@ internal static List> ConvertToJobServiceCont internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, TemplateToken token) { string imageName = null; + string version = "1.*"; + string versionString = string.Empty; + var condition = $"{PipelineTemplateConstants.Success}()"; + if (token is StringToken snapshotStringLiteral) { imageName = snapshotStringLiteral.Value; @@ -359,11 +363,19 @@ internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, Te foreach (var snapshotPropertyPair in snapshotMapping) { var propertyName = snapshotPropertyPair.Key.AssertString($"{PipelineTemplateConstants.Snapshot} key"); + var propertyValue = snapshotPropertyPair.Value; switch (propertyName.Value) { case PipelineTemplateConstants.ImageName: imageName = snapshotPropertyPair.Value.AssertString($"{PipelineTemplateConstants.Snapshot} {propertyName}").Value; break; + case PipelineTemplateConstants.If: + condition = ConvertToIfCondition(context, propertyValue, false); + break; + case PipelineTemplateConstants.CustomImageVersion: + versionString = propertyValue.AssertString($"job {PipelineTemplateConstants.Snapshot} {PipelineTemplateConstants.CustomImageVersion}").Value; + version = IsSnapshotImageVersionValid(versionString) ? versionString : null; + break; default: propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Snapshot} key"); break; @@ -376,7 +388,26 @@ internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, Te return null; } - return new Snapshot(imageName); + return new Snapshot(imageName) + { + Condition = condition, + Version = version + }; + } + + private static bool IsSnapshotImageVersionValid(string versionString) + { + var versionSegments = versionString.Split("."); + + if (versionSegments.Length != 2 || + !versionSegments[1].Equals("*") || + !Int32.TryParse(versionSegments[0], result: out int parsedMajor) || + parsedMajor <= 0) + { + return false; + } + + return true; } private static ActionStep ConvertToStep( diff --git a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs index 60f8da04f4f..ffe00430b0a 100644 --- a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs +++ b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.Serialization; +using GitHub.DistributedTask.ObjectTemplating.Tokens; namespace GitHub.DistributedTask.Pipelines { @@ -13,5 +14,15 @@ public Snapshot(string imageName) [DataMember(EmitDefaultValue = false)] public String ImageName { get; set; } + + [DataMember(EmitDefaultValue = false)] + public String Condition + { + get; + set; + } + + [DataMember(EmitDefaultValue = false)] + public String Version { get; set; } } } diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index a3837edff02..bc706eb33ad 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -162,18 +162,35 @@ "snapshot-mapping" ] }, - + "snapshot-mapping": { "mapping": { "properties": { "image-name": { "type": "non-empty-string", "required": true + }, + "if": "snapshot-if", + "version": { + "type": "non-empty-string", + "required": false } } } }, + "snapshot-if": { + "context": [ + "github", + "inputs", + "vars", + "needs", + "strategy", + "matrix" + ], + "string": {} + }, + "runs-on": { "context": [ "github", diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 4f5834fbe69..220fa065102 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -506,7 +506,30 @@ public Task EnsureSnapshotPostJobStepForMappingToken() return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot); } - private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot) + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public Task EnsureSnapshotPostJobStepForMappingToken_1() + { + var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken"){ + Condition = $"{PipelineTemplateConstants.Success}() && 1==0", + Version = "2.*" + }; + var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName); + var condition = new StringToken(null, null, null, snapshot.Condition); + var version = new StringToken(null, null, null, snapshot.Version); + + var mappingToken = new MappingToken(null, null, null) + { + { new StringToken(null,null,null, PipelineTemplateConstants.ImageName), imageNameValueStringToken }, + { new StringToken(null,null,null, PipelineTemplateConstants.If), condition }, + { new StringToken(null,null,null, PipelineTemplateConstants.CustomImageVersion), version } + }; + + return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot, skipSnapshotStep: true); + } + + private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot, bool skipSnapshotStep = false) { using (TestHostContext hc = CreateTestContext()) { @@ -524,14 +547,28 @@ private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken Assert.Equal(1, postJobSteps.Count); var snapshotStep = postJobSteps.First(); + _jobEc.JobSteps.Enqueue(snapshotStep); + + var _stepsRunner = new StepsRunner(); + _stepsRunner.Initialize(hc); + await _stepsRunner.RunAsync(_jobEc); + Assert.Equal("Create custom image", snapshotStep.DisplayName); - Assert.Equal($"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); + Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); // Run the mock snapshot step, so we can verify it was executed with the expected snapshot object. - await snapshotStep.RunAsync(); - - Assert.NotNull(_requestedSnapshot); - Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName); + // await snapshotStep.RunAsync(); + if (skipSnapshotStep) + { + Assert.Null(_requestedSnapshot); + } + else + { + Assert.NotNull(_requestedSnapshot); + Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName); + Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", _requestedSnapshot.Condition); + Assert.Equal(expectedSnapshot.Version ?? "1.*", _requestedSnapshot.Version); + } } } } From ba8305ef0a526024a0f41bcd26908f43df08915e Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Thu, 19 Sep 2024 00:57:51 +0000 Subject: [PATCH 2/7] Lint failure fixes --- .../Pipelines/ObjectTemplating/PipelineTemplateConverter.cs | 5 +---- src/Test/L0/Worker/JobExtensionL0.cs | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 9105dc40a33..d24c4ad8b8a 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -399,10 +399,7 @@ private static bool IsSnapshotImageVersionValid(string versionString) { var versionSegments = versionString.Split("."); - if (versionSegments.Length != 2 || - !versionSegments[1].Equals("*") || - !Int32.TryParse(versionSegments[0], result: out int parsedMajor) || - parsedMajor <= 0) + if (versionSegments.Length != 2 || !versionSegments[1].Equals("*") || !Int32.TryParse(versionSegments[0], result: out int parsedMajor) || parsedMajor <= 0) { return false; } diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 220fa065102..66bc0383402 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -506,12 +506,12 @@ public Task EnsureSnapshotPostJobStepForMappingToken() return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot); } - [Fact] + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] public Task EnsureSnapshotPostJobStepForMappingToken_1() { - var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken"){ + var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken") { Condition = $"{PipelineTemplateConstants.Success}() && 1==0", Version = "2.*" }; @@ -552,7 +552,7 @@ private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken var _stepsRunner = new StepsRunner(); _stepsRunner.Initialize(hc); await _stepsRunner.RunAsync(_jobEc); - + Assert.Equal("Create custom image", snapshotStep.DisplayName); Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); From f5d96d09e47ccdc352ebd81ba30d7d1dd4069372 Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Thu, 19 Sep 2024 01:02:24 +0000 Subject: [PATCH 3/7] Lint failure fixes - 2 --- src/Sdk/DTPipelines/Pipelines/Snapshot.cs | 8 ++------ src/Test/L0/Worker/JobExtensionL0.cs | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs index ffe00430b0a..a389b39e85d 100644 --- a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs +++ b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs @@ -16,13 +16,9 @@ public Snapshot(string imageName) public String ImageName { get; set; } [DataMember(EmitDefaultValue = false)] - public String Condition - { - get; - set; - } + public String Condition { get; set; } [DataMember(EmitDefaultValue = false)] - public String Version { get; set; } + public String Version { get; set; } } } diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 66bc0383402..c46c13cc1ef 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -511,7 +511,7 @@ public Task EnsureSnapshotPostJobStepForMappingToken() [Trait("Category", "Worker")] public Task EnsureSnapshotPostJobStepForMappingToken_1() { - var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken") { + var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken"){ Condition = $"{PipelineTemplateConstants.Success}() && 1==0", Version = "2.*" }; From d549a8bb128c12b8ae57855cd72ef05bb8ed066b Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Thu, 19 Sep 2024 01:06:50 +0000 Subject: [PATCH 4/7] Lint failure fixes - 3 --- src/Sdk/DTPipelines/Pipelines/Snapshot.cs | 5 ++++- src/Test/L0/Worker/JobExtensionL0.cs | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs index a389b39e85d..c1a05674aea 100644 --- a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs +++ b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs @@ -1,15 +1,18 @@ using System; using System.Runtime.Serialization; using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ObjectTemplating; namespace GitHub.DistributedTask.Pipelines { [DataContract] public class Snapshot { - public Snapshot(string imageName) + public Snapshot(string imageName, string condition = null, string version = null) { ImageName = imageName; + Condition = condition ?? $"{PipelineTemplateConstants.Success}()"; + Version = version ?? "1.*"; } [DataMember(EmitDefaultValue = false)] diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index c46c13cc1ef..e8c6f760b66 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -511,10 +511,7 @@ public Task EnsureSnapshotPostJobStepForMappingToken() [Trait("Category", "Worker")] public Task EnsureSnapshotPostJobStepForMappingToken_1() { - var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken"){ - Condition = $"{PipelineTemplateConstants.Success}() && 1==0", - Version = "2.*" - }; + var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken", condition: $"{PipelineTemplateConstants.Success}() && 1==0", version: "2.*"); var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName); var condition = new StringToken(null, null, null, snapshot.Condition); var version = new StringToken(null, null, null, snapshot.Version); From db70cee66547449d8e7ae0ac378525793ad7df1c Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Tue, 1 Oct 2024 00:10:58 +0000 Subject: [PATCH 5/7] Fixed a few nits --- .../Pipelines/ObjectTemplating/PipelineTemplateConverter.cs | 6 +++++- src/Test/L0/Worker/JobExtensionL0.cs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index d24c4ad8b8a..181a6a2350b 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Globalization; using System.Linq; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2.Sdk; @@ -399,7 +400,10 @@ private static bool IsSnapshotImageVersionValid(string versionString) { var versionSegments = versionString.Split("."); - if (versionSegments.Length != 2 || !versionSegments[1].Equals("*") || !Int32.TryParse(versionSegments[0], result: out int parsedMajor) || parsedMajor <= 0) + if (versionSegments.Length != 2 || + !versionSegments[1].Equals("*") || + !Int32.TryParse(versionSegments[0], NumberStyles.None, CultureInfo.InvariantCulture, result: out int parsedMajor) || + parsedMajor < 0) { return false; } diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index e8c6f760b66..5db32d47a30 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -509,7 +509,7 @@ public Task EnsureSnapshotPostJobStepForMappingToken() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public Task EnsureSnapshotPostJobStepForMappingToken_1() + public Task EnsureSnapshotPostJobStepForMappingToken_WithIf_Is_False() { var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken", condition: $"{PipelineTemplateConstants.Success}() && 1==0", version: "2.*"); var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName); From 2ae0914c44ac90b13b931c637a618afbf02bcb2d Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Tue, 1 Oct 2024 00:14:49 +0000 Subject: [PATCH 6/7] Lint fixes --- .../Pipelines/ObjectTemplating/PipelineTemplateConverter.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 181a6a2350b..40f6a13345f 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -400,9 +400,9 @@ private static bool IsSnapshotImageVersionValid(string versionString) { var versionSegments = versionString.Split("."); - if (versionSegments.Length != 2 || - !versionSegments[1].Equals("*") || - !Int32.TryParse(versionSegments[0], NumberStyles.None, CultureInfo.InvariantCulture, result: out int parsedMajor) || + if (versionSegments.Length != 2 || + !versionSegments[1].Equals("*") || + !Int32.TryParse(versionSegments[0], NumberStyles.None, CultureInfo.InvariantCulture, result: out int parsedMajor) || parsedMajor < 0) { return false; From 76098b0349d23227d2d3a689a79bac7dc39f9fb3 Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:57:00 +0000 Subject: [PATCH 7/7] Removed unncessary white space --- src/Sdk/DTPipelines/workflow-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index bc706eb33ad..ec09cfe58b7 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -162,7 +162,7 @@ "snapshot-mapping" ] }, - + "snapshot-mapping": { "mapping": { "properties": {