diff --git a/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj b/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj index 81f69826d59..9957e6ba219 100644 --- a/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj +++ b/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj @@ -26,7 +26,7 @@ - + @@ -45,12 +45,14 @@ - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + + diff --git a/src/dotnet/APIView/APIViewWeb/Controllers/PullRequestController.cs b/src/dotnet/APIView/APIViewWeb/Controllers/PullRequestController.cs index a52883fb139..0b5d9dea013 100644 --- a/src/dotnet/APIView/APIViewWeb/Controllers/PullRequestController.cs +++ b/src/dotnet/APIView/APIViewWeb/Controllers/PullRequestController.cs @@ -63,7 +63,8 @@ public async Task DetectApiChanges( string codeFile = null, string baselineCodeFile = null, bool commentOnPR = true, - string language = null) + string language = null, + string project = "internal") { if (!ValidateInputParams()) { @@ -80,7 +81,7 @@ public async Task DetectApiChanges( repoName: repoName, packageName: packageName, prNumber: pullRequestNumber, hostName: this.Request.Host.ToUriComponent(), codeFileName: codeFile, baselineCodeFileName: baselineCodeFile, - commentOnPR: commentOnPR, language: language); + commentOnPR: commentOnPR, language: language, project: project); return !string.IsNullOrEmpty(reviewUrl) ? StatusCode(statusCode: StatusCodes.Status201Created, reviewUrl) : StatusCode(statusCode: StatusCodes.Status208AlreadyReported); } @@ -102,7 +103,7 @@ private async Task DetectAPIChanges(string buildId, string baselineCodeFileName = null, bool commentOnPR = true, string language = null, - string project = "public") + string project = "internal") { language = LanguageServiceHelpers.MapLanguageAlias(language: language); var requestTelemetry = new RequestTelemetry { Name = "Detecting API changes for PR: " + prNumber }; diff --git a/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs b/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs index fbaf660f9b2..012c1a90091 100644 --- a/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs +++ b/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs @@ -1,130 +1,111 @@ -using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.ApplicationInsights; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Caching.Memory; +using Azure.Core; +using Azure.Identity; +using Microsoft.ApplicationInsights; using Microsoft.Extensions.Configuration; -using Microsoft.TeamFoundation.Build.WebApi; -using Microsoft.TeamFoundation.Core.WebApi; -using Microsoft.VisualStudio.Services.Common; -using Microsoft.VisualStudio.Services.WebApi; -using Newtonsoft.Json; -using Octokit; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.TeamFoundation.Core.WebApi; +using Microsoft.VisualStudio.Services.Client; +using Microsoft.VisualStudio.Services.WebApi; +using Newtonsoft.Json; +using Polly; using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Net.Http; using System.Net.Http.Headers; -using System.Text.Json; +using System.Threading; using System.Threading.Tasks; namespace APIViewWeb.Repositories { public class DevopsArtifactRepository : IDevopsArtifactRepository { - private readonly HttpClient _devopsClient; private readonly IConfiguration _configuration; - private readonly string _devopsAccessToken; private readonly string _hostUrl; private readonly TelemetryClient _telemetryClient; public DevopsArtifactRepository(IConfiguration configuration, TelemetryClient telemetryClient) { _configuration = configuration; - _devopsAccessToken = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", _configuration["Azure-Devops-PAT"]))); _hostUrl = _configuration["APIVIew-Host-Url"]; _telemetryClient = telemetryClient; - - _devopsClient = new HttpClient(); - _devopsClient.DefaultRequestHeaders.Accept.Clear(); - _devopsClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - _devopsClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", _devopsAccessToken); } public async Task DownloadPackageArtifact(string repoName, string buildId, string artifactName, string filePath, string project, string format= "file") { - var downloadUrl = await GetDownloadArtifactUrl(repoName, buildId, artifactName, project); - if (!string.IsNullOrEmpty(downloadUrl)) + var downloadUrl = await getDownloadArtifactUrl(buildId, artifactName, project); + if (string.IsNullOrEmpty(downloadUrl)) + { + throw new Exception(string.Format("Failed to get download url for artifact {0} in build {1} in project {2}", artifactName, buildId, project)); + } + + if(!string.IsNullOrEmpty(filePath)) { - if(!string.IsNullOrEmpty(filePath)) + if (!filePath.StartsWith("/")) { - if (!filePath.StartsWith("/")) - { - filePath = "/" + filePath; - } - downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath; + filePath = "/" + filePath; } - - var downloadResp = await GetFromDevopsAsync(downloadUrl); - downloadResp.EnsureSuccessStatusCode(); - return await downloadResp.Content.ReadAsStreamAsync(); + downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath; } - return null; + + HttpResponseMessage downloadResp = await GetFromDevopsAsync(downloadUrl); + downloadResp.EnsureSuccessStatusCode(); + return await downloadResp.Content.ReadAsStreamAsync(); } - private async Task GetFromDevopsAsync(string request) + private async Task getDownloadArtifactUrl(string buildId, string artifactName, string project) { - var downloadResp = await _devopsClient.GetAsync(request); - - - if (!downloadResp.IsSuccessStatusCode) - { - var retryAfter = downloadResp.Headers.GetValues("Retry-After"); - var rateLimitResource = downloadResp.Headers.GetValues("X-RateLimit-Resource"); - var rateLimitDelay = downloadResp.Headers.GetValues("X-RateLimit-Delay"); - var rateLimitLimit = downloadResp.Headers .GetValues("X-RateLimit-Limit"); - var rateLimitRemaining = downloadResp.Headers.GetValues("X-RateLimit-Remaining"); - var rateLimitReset = downloadResp.Headers.GetValues("X-RateLimit-Reset"); - - var traceMessage = $"request: {request} failed with statusCode: {downloadResp.StatusCode}," + - $"retryAfter: {retryAfter.FirstOrDefault()}, rateLimitResource: {rateLimitResource.FirstOrDefault()}, rateLimitDelay: {rateLimitDelay.FirstOrDefault()}," + - $"rateLimitLimit: {rateLimitLimit.FirstOrDefault()}, rateLimitRemaining: {rateLimitRemaining.FirstOrDefault()}, rateLimitReset: {rateLimitReset.FirstOrDefault()}"; - - _telemetryClient.TrackTrace(traceMessage); - } + var connection = await CreateVssConnection(); + var buildClient = connection.GetClient(); + var artifact = await buildClient.GetArtifactAsync(project, int.Parse(buildId), artifactName); + return artifact?.Resource?.DownloadUrl; + } - int count = 0; - int[] waitTimes = new int[] { 0, 1, 2, 4, 8, 16, 32, 64, 128, 256 }; - while ((downloadResp.StatusCode == HttpStatusCode.TooManyRequests || downloadResp.StatusCode == HttpStatusCode.BadRequest) && count < waitTimes.Length) - { - _telemetryClient.TrackTrace($"Download request from devops artifact is either throttled or flaky, waiting {waitTimes[count]} seconds before retrying, Retry count: {count}"); - await Task.Delay(TimeSpan.FromSeconds(waitTimes[count])); - downloadResp = await _devopsClient.GetAsync(request); - count++; - } - return downloadResp; + private async Task CreateVssConnection() + { + var accessToken = await getAccessToken(); + var token = new VssAadToken("Bearer", accessToken); + return new VssConnection(new Uri("https://dev.azure.com/azure-sdk/"), new VssAadCredential(token)); } - private async Task GetDownloadArtifactUrl(string repoName, string buildId, string artifactName, string project) + private async Task getAccessToken() { - var artifactGetReq = GetArtifactRestAPIForRepo(repoName).Replace("{buildId}", buildId).Replace("{artifactName}", artifactName).Replace("{project}", project); - var response = await GetFromDevopsAsync(artifactGetReq); - response.EnsureSuccessStatusCode(); - var buildResource = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); - if (buildResource == null) - { - return null; - } - return buildResource.RootElement.GetProperty("resource").GetProperty("downloadUrl").GetString(); + // APIView deployed instances uses managed identity to authenticate requests to Azure DevOps. + // For local testing, VS will use developer credentials to create token + var credential = new DefaultAzureCredential(); + var tokenRequestContext = new TokenRequestContext(VssAadSettings.DefaultScopes); + var token = await credential.GetTokenAsync(tokenRequestContext, CancellationToken.None); + return token.Token; } - private string GetArtifactRestAPIForRepo(string repoName) + private async Task GetFromDevopsAsync(string request) { - var downloadArtifactRestApi = _configuration["download-artifact-rest-api-for-" + repoName]; - if (downloadArtifactRestApi == null) + var httpClient = new HttpClient(); + var accessToken = await getAccessToken(); + httpClient.DefaultRequestHeaders.Accept.Clear(); + httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + var maxRetryAttempts = 10; + var pauseBetweenFailures = TimeSpan.FromSeconds(2); + + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(maxRetryAttempts, i => pauseBetweenFailures); + + HttpResponseMessage downloadResp = null; + await retryPolicy.ExecuteAsync(async () => { - downloadArtifactRestApi = _configuration["download-artifact-rest-api"]; - } - return downloadArtifactRestApi; + downloadResp = await httpClient.GetAsync(request); + }); + return downloadResp; } public async Task RunPipeline(string pipelineName, string reviewDetails, string originalStorageUrl) { //Create dictionary of all required parametes to run tools - generate--apireview pipeline in azure devops var reviewDetailsDict = new Dictionary { { "Reviews", reviewDetails }, { "APIViewUrl", _hostUrl }, { "StorageContainerUrl", originalStorageUrl } }; - var devOpsCreds = new VssBasicCredential("nobody", _configuration["Azure-Devops-PAT"]); - var devOpsConnection = new VssConnection(new Uri($"https://dev.azure.com/azure-sdk/"), devOpsCreds); + var devOpsConnection = await CreateVssConnection(); string projectName = _configuration["Azure-Devops-internal-project"] ?? "internal"; BuildHttpClient buildClient = await devOpsConnection.GetClientAsync(); @@ -132,29 +113,29 @@ public async Task RunPipeline(string pipelineName, string reviewDetails, string string envName = _configuration["apiview-deployment-environment"]; string updatedPipelineName = string.IsNullOrEmpty(envName) ? pipelineName : $"{pipelineName}-{envName}"; int definitionId = await GetPipelineId(updatedPipelineName, buildClient, projectName); - if (definitionId == 0) - { - throw new Exception(string.Format("Azure Devops pipeline is not found with name {0}. Please recheck and ensure pipeline exists with this name", updatedPipelineName)); + if (definitionId == 0) + { + throw new Exception(string.Format("Azure Devops pipeline is not found with name {0}. Please recheck and ensure pipeline exists with this name", updatedPipelineName)); } - var definition = await buildClient.GetDefinitionAsync(projectName, definitionId); + var definition = await buildClient.GetDefinitionAsync(projectName, definitionId); var project = await projectClient.GetProject(projectName); - await buildClient.QueueBuildAsync(new Build() - { - Definition = definition, - Project = project, - Parameters = JsonConvert.SerializeObject(reviewDetailsDict) + await buildClient.QueueBuildAsync(new Build() + { + Definition = definition, + Project = project, + Parameters = JsonConvert.SerializeObject(reviewDetailsDict) }); } - private async Task GetPipelineId(string pipelineName, BuildHttpClient client, string projectName) - { - var pipelines = await client.GetFullDefinitionsAsync2(project: projectName); - if (pipelines != null) - { - return pipelines.Single(p => p.Name == pipelineName).Id; - } - return 0; + private async Task GetPipelineId(string pipelineName, BuildHttpClient client, string projectName) + { + var pipelines = await client.GetFullDefinitionsAsync2(project: projectName); + if (pipelines != null) + { + return pipelines.Single(p => p.Name == pipelineName).Id; + } + return 0; } } }