Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add task to download files from helix results container #2103

Merged
merged 5 commits into from
Feb 27, 2019
Merged

Add task to download files from helix results container #2103

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2e31c0b
Add task to download files from helix results container
safern Feb 24, 2019
b0fcfd9
PR Feedback
safern Feb 26, 2019
ae6703b
Fix warning conditions to only be emitted when feature is ON
safern Feb 26, 2019
85c8b65
Implement ICancelableTask
safern Feb 27, 2019
3e862cb
Add more detailed warning when blob not found
safern Feb 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ public async Task<AgentInfoItem> CreateJob()
.WithContainerName(_configuration.ContainerName)
.WithCorrelationPayloadUris(AgentPayloadUri)
.WithStorageAccountConnectionString(_configuration.ConnectionString)
.WithResultsStorageAccountConnectionString(_configuration.ConnectionString)
.DefineWorkItem(_agentRequestItem.agentId)
.WithCommand(ConstructCommand())
.WithFiles(credentialsPath, agentSettingsPath, StartupScriptPath)
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@ namespace Microsoft.DotNet.Helix.Client
public interface ISentJob
{
string CorrelationId { get; }
string ResultsContainerUri { get; }
}
}
21 changes: 12 additions & 9 deletions src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,15 @@ public async Task<ISentJob> SendAsync(Action<string> log = null)
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List<JobListEntry>();

IBlobHelper resultsStorage = null;
IBlobContainer resultsStorageContainer = null;
if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
IBlobContainer resultsStorageContainer;
if (string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
{
resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetContainerName);
resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
safern marked this conversation as resolved.
Show resolved Hide resolved
}
else
{
IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
}

List<string> correlationPayloadUris =
Expand Down Expand Up @@ -188,14 +191,14 @@ public async Task<ISentJob> SendAsync(Action<string> log = null)
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
ResultsUri = resultsStorageContainer?.Uri,
ResultsUriRSAS = resultsStorageContainer?.ReadSas,
ResultsUriWSAS = resultsStorageContainer?.WriteSas,
ResultsUri = resultsStorageContainer.Uri,
ResultsUriRSAS = resultsStorageContainer.ReadSas,
ResultsUriWSAS = resultsStorageContainer.WriteSas,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));


return new SentJob(JobApi, newJob);
return new SentJob(JobApi, newJob, resultsStorageContainer.Uri);
}

public IJobDefinitionWithTargetQueue WithBuild(string buildNumber)
Expand Down
4 changes: 3 additions & 1 deletion src/Microsoft.DotNet.Helix/JobSender/SentJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ namespace Microsoft.DotNet.Helix.Client
{
internal class SentJob : ISentJob
{
public SentJob(IJob jobApi, JobCreationResult newJob)
public SentJob(IJob jobApi, JobCreationResult newJob, string resultsContainerUri)
{
JobApi = jobApi;
CorrelationId = newJob.Name;
ResultsContainerUri = resultsContainerUri;
}

public IJob JobApi { get; }
public string CorrelationId { get; }
public string ResultsContainerUri { get; }
}
}
104 changes: 104 additions & 0 deletions src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
using Microsoft.Build.Framework;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace Microsoft.DotNet.Helix.Sdk
{
public class DownloadFromResultsContainer : BaseTask
safern marked this conversation as resolved.
Show resolved Hide resolved
{
[Required]
public ITaskItem[] WorkItems { get; set; }

[Required]
public string ResultsContainer { get; set; }

[Required]
public string PathToDownload { get; set; }
safern marked this conversation as resolved.
Show resolved Hide resolved

[Required]
public string JobId { get; set; }

[Required]
public ITaskItem[] MetadataToWrite { get; set; }

private const string MetadataFile = "metadata.txt";

public override bool Execute()
{
if (string.IsNullOrEmpty(ResultsContainer))
{
LogRequiredParameterError(nameof(ResultsContainer));
}

if (string.IsNullOrEmpty(PathToDownload))
{
LogRequiredParameterError(nameof(PathToDownload));
}

if (string.IsNullOrEmpty(JobId))
{
LogRequiredParameterError(nameof(JobId));
}

if (!Log.HasLoggedErrors)
{
Log.LogMessage(MessageImportance.High, $"Downloading result files for job {JobId}");
ExecuteCore().GetAwaiter().GetResult();
}

return !Log.HasLoggedErrors;
}

private async Task ExecuteCore()
{
DirectoryInfo directory = Directory.CreateDirectory(Path.Combine(PathToDownload, JobId));
using (FileStream stream = File.Open(Path.Combine(directory.FullName, MetadataFile), FileMode.Create, FileAccess.Write))
using (var writer = new StreamWriter(stream))
{
foreach (ITaskItem metadata in MetadataToWrite)
{
await writer.WriteLineAsync(metadata.GetMetadata("Identity"));
}
}

ResultsContainer = ResultsContainer.EndsWith("/") ? ResultsContainer : ResultsContainer + "/";
await Task.WhenAll(WorkItems.Select(wi => DownloadFilesForWorkItem(wi, directory.FullName)));
return;
}

private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath)
{
if (workItem.GetRequiredMetadata(Log, "DownloadFilesFromResults", out string files))
safern marked this conversation as resolved.
Show resolved Hide resolved
{
string workItemName = workItem.GetMetadata("Identity");
string[] filesToDownload = files.Split(';');

DirectoryInfo destinationDir = Directory.CreateDirectory(Path.Combine(directoryPath, workItemName));
foreach (var file in filesToDownload)
{
try
{
string destinationFile = Path.Combine(destinationDir.FullName, file);
Log.LogMessage(MessageImportance.Normal, $"Downloading {file} => {destinationFile}...");
CloudBlob blob = new CloudBlob(new Uri($"{ResultsContainer}{workItemName}/{file}"));
safern marked this conversation as resolved.
Show resolved Hide resolved
await blob.DownloadToFileAsync(destinationFile, FileMode.Create);
}
catch (StorageException e)
{
Log.LogWarning($"Failed to download {file} from results container: {e.Message}");
}
}
};
return;
}

private void LogRequiredParameterError(string parameter)
{
Log.LogError($"Required parameter {nameof(parameter)} string was null or empty");
safern marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
7 changes: 7 additions & 0 deletions src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ public class SendHelixJob : HelixTask
[Output]
public string JobCorrelationId { get; set; }

/// <summary>
/// When the task finishes, the results container uri should be available in case we want to download files.
/// </summary>
[Output]
public string ResultsContainerUri { get; set; }

/// <summary>
/// A collection of commands that will run for each work item before any work item commands.
/// Use ';' to separate commands and escape a ';' with ';;'
Expand Down Expand Up @@ -209,6 +215,7 @@ protected override async Task ExecuteCore()

ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg));
JobCorrelationId = job.CorrelationId;
ResultsContainerUri = job.ResultsContainerUri;
}

string mcUri = await GetMissionControlResultUri();
Expand Down
3 changes: 3 additions & 0 deletions src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,13 @@
WorkItems="@(HelixWorkItem)"
HelixProperties="@(HelixProperties)">
<Output TaskParameter="JobCorrelationId" PropertyName="HelixJobId"/>
<Output TaskParameter="ResultsContainerUri" PropertyName="HelixResultsContainer"/>
</SendHelixJob>
<ItemGroup>
<SentJob Include="$(HelixJobId)">
<WorkItemCount>@(HelixWorkItem->Count())</WorkItemCount>
<HelixTargetQueue>$(HelixTargetQueue)</HelixTargetQueue>
<ResultsContainerUri>$(HelixResultsContainer)</ResultsContainerUri>
</SentJob>
</ItemGroup>
<Message Text="Sent Helix Job $(HelixJobId)" Importance="High" />
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,5 @@
<UsingTask TaskName="StartAzurePipelinesTestRun" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>
<UsingTask TaskName="StopAzurePipelinesTestRun" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>
<UsingTask TaskName="CheckAzurePipelinesTestRun" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>
<UsingTask TaskName="DownloadFromResultsContainer" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>
</Project>
5 changes: 5 additions & 0 deletions src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.props
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
<Project>
<PropertyGroup>
<_HelixMultiQueueTargets>$(_HelixMultiQueueTargets);$(MSBuildThisFileDirectory)DownloadFromResultsContainer.targets</_HelixMultiQueueTargets>
</PropertyGroup>
</Project>
30 changes: 30 additions & 0 deletions src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
<Project>
<Target Name="DownloadFromResultsContainer"
Condition="$(WaitForWorkItemCompletion)"
AfterTargets="CoreTest"
Inputs="unused"
Outputs="%(SentJob.HelixTargetQueue);%(SentJob.ResultsContainerUri)">
safern marked this conversation as resolved.
Show resolved Hide resolved
<ItemGroup>
<_workItemsWithDownloadMetadata Include="@(HelixWorkItem)" Condition="'%(HelixWorkItem.DownloadFilesFromResults)' != ''" />
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you actually need to empty these item groups before adding things to them. IIRC this will currently write the metadata for jobs 1 and 2 together in the file for 2 because the item group has been filled with both.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't seen this behavior locally, but it doesn't hurt to empty this item group.

Actually the metadata itemgroup was correctly written for multiple jobs. No duplication at all.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hrm, I wonder if there is some msbuild magic with batching that is making the items not show up.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe because the ItemGroup is within the scope of the target and since we're batching it creates 2 different copies and evaluations of it? Basically 2 different scopes? @ericstj might know.

Copy link
Member

@ericstj ericstj Feb 26, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the side-effects of a batched target aren't observable until after all batches of that target run.

Try this:

<Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
    <OutputType>Exe</OutputType>
    <TargetFramework>netcoreapp3.0</TargetFramework>
    <_newProperty>unset</_newProperty>
  </PropertyGroup>

  <ItemGroup>
    <TestItem Include="Abc" />
    <TestItem Include="123" />
  </ItemGroup>
  
  <Target Name="BatchTarget" Inputs="%(TestItem.Identity)" Outputs="Unused">
    <Message Text="BatchTarget Property: $(_newProperty)" Importance="High" />
    <ItemGroup>
      <_newItem Include="@(TestItem)" />
    </ItemGroup>
    <PropertyGroup>
      <_newProperty>@(TestItem)</_newProperty>
    </PropertyGroup>
    <Message Text="BatchTarget: @(_newItem)" Importance="High" />
  </Target>
  
  <Target Name="TestTarget" DependsOnTargets="BatchTarget">
    <Message Text="TestTarget Property: $(_newProperty)" Importance="High" />
    <Message Text="TestTarget: @(_newItem)" Importance="High" />
  </Target>

</Project>

Output of this is:

  BatchTarget Property: unset
  BatchTarget: Abc
  BatchTarget Property: unset
  BatchTarget: 123
  TestTarget Property: 123
  TestTarget: Abc;123

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we don't intend to use this items outside the target I guess we're safe on leaving it as is.


<DownloadResultsMetadata Include="HelixQueue=%(SentJob.HelixTargetQueue)" />
<DownloadResultsMetadata Condition="'$(HelixConfiguration)' != ''" Include="Configuration=$(HelixConfiguration)" />
<DownloadResultsMetadata Condition="'$(HelixArchitecture)' != ''" Include="Architecture=$(HelixArchitecture)" />
</ItemGroup>

<PropertyGroup>
<ResultsDestinationPath Condition="'$(ResultsDestinationPath)' == '' AND '$(BUILD_SOURCESDIRECTORY)' != ''">$([MSBuild]::NormalizePath('$(BUILD_SOURCESDIRECTORY)', 'artifacts', helixresults'))</ResultsDestinationPath>
<ResultsDestinationPath Condition="'$(ResultsDestinationPath)' == ''">$([MSBuild]::NormalizePath('$(MSBuildStartupDirectory)', 'helixresults'))</ResultsDestinationPath>
safern marked this conversation as resolved.
Show resolved Hide resolved
</PropertyGroup>

<Warning Text="DownloadFromResultsContainer will be skipped for job %(SentJob.Identity) becaue results container uri is empty" Condition="'%(SentJob.ResultsContainerUri)' == ''" />

<DownloadFromResultsContainer
Condition="'@(_workItemsWithDownloadMetadata)' != '' AND '%(SentJob.ResultsContainerUri)' != ''"
WorkItems="@(_workItemsWithDownloadMetadata)"
ResultsContainer="%(SentJob.ResultsContainerUri)"
PathToDownload="$(ResultsDestinationPath)"
MetadataToWrite="@(DownloadResultsMetadata)"
JobId="%(SentJob.Identity)" />
</Target>
</Project>