Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Authenticate requests from APIView to DevOps using managed Identity #8086

Merged
merged 3 commits into from
Apr 16, 2024
Merged

Authenticate requests from APIView to DevOps using managed Identity #8086

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c05a14e
Authenticate requests from APIView to DevOps using managed Identity
praveenkuttappan Apr 13, 2024
099f89b
Update default devops project name to internal to download artifact
praveenkuttappan Apr 13, 2024
b5af2b7
use identity token to download content
praveenkuttappan Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" />
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.6" />
<PackageReference Include="Azure.Data.AppConfiguration" Version="1.3.0" />
<PackageReference Include="Azure.Identity" Version="1.10.4" />
<PackageReference Include="Azure.Identity" Version="1.11.0" />
<PackageReference Include="Azure.Search.Documents" Version="11.5.0-beta.4" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
<PackageReference Include="CsvHelper" Version="30.0.1" />
Expand All @@ -45,12 +45,14 @@
<PackageReference Include="Microsoft.Azure.AppConfiguration.AspNetCore" Version="7.0.0" />
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.37.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="7.0.15" />
<PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.205.1" />
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.4" />
<PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="19.225.1" />
<PackageReference Include="Microsoft.TypeScript.MSBuild" Version="5.3.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.205.1" />
<PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="19.225.1" />
<PackageReference Include="Microsoft.VisualStudio.Services.InteractiveClient" Version="19.225.1" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.11" PrivateAssets="All" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0-beta2-19367-01" PrivateAssets="All" />
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
Expand Down
7 changes: 4 additions & 3 deletions src/dotnet/APIView/APIViewWeb/Controllers/PullRequestController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ public async Task<ActionResult> DetectApiChanges(
string codeFile = null,
string baselineCodeFile = null,
bool commentOnPR = true,
string language = null)
string language = null,
string project = "internal")
{
if (!ValidateInputParams())
{
Expand All @@ -80,7 +81,7 @@ public async Task<ActionResult> DetectApiChanges(
repoName: repoName, packageName: packageName,
prNumber: pullRequestNumber, hostName: this.Request.Host.ToUriComponent(),
codeFileName: codeFile, baselineCodeFileName: baselineCodeFile,
commentOnPR: commentOnPR, language: language);
commentOnPR: commentOnPR, language: language, project: project);

return !string.IsNullOrEmpty(reviewUrl) ? StatusCode(statusCode: StatusCodes.Status201Created, reviewUrl) : StatusCode(statusCode: StatusCodes.Status208AlreadyReported);
}
Expand All @@ -102,7 +103,7 @@ private async Task<string> DetectAPIChanges(string buildId,
string baselineCodeFileName = null,
bool commentOnPR = true,
string language = null,
string project = "public")
string project = "internal")
{
language = LanguageServiceHelpers.MapLanguageAlias(language: language);
var requestTelemetry = new RequestTelemetry { Name = "Detecting API changes for PR: " + prNumber };
Expand Down
175 changes: 78 additions & 97 deletions src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,160 +1,141 @@
using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.ApplicationInsights;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Caching.Memory;
using Azure.Core;
using Azure.Identity;
using Microsoft.ApplicationInsights;
using Microsoft.Extensions.Configuration;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.TeamFoundation.Core.WebApi;
using Microsoft.VisualStudio.Services.Common;
using Microsoft.VisualStudio.Services.WebApi;
using Newtonsoft.Json;
using Octokit;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.TeamFoundation.Core.WebApi;
using Microsoft.VisualStudio.Services.Client;
using Microsoft.VisualStudio.Services.WebApi;
using Newtonsoft.Json;
using Polly;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;

namespace APIViewWeb.Repositories
{
public class DevopsArtifactRepository : IDevopsArtifactRepository
{
private readonly HttpClient _devopsClient;
private readonly IConfiguration _configuration;
private readonly string _devopsAccessToken;
private readonly string _hostUrl;
private readonly TelemetryClient _telemetryClient;

public DevopsArtifactRepository(IConfiguration configuration, TelemetryClient telemetryClient)
{
_configuration = configuration;
_devopsAccessToken = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", _configuration["Azure-Devops-PAT"])));
_hostUrl = _configuration["APIVIew-Host-Url"];
_telemetryClient = telemetryClient;

_devopsClient = new HttpClient();
_devopsClient.DefaultRequestHeaders.Accept.Clear();
_devopsClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
_devopsClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", _devopsAccessToken);
}

public async Task<Stream> DownloadPackageArtifact(string repoName, string buildId, string artifactName, string filePath, string project, string format= "file")
{
var downloadUrl = await GetDownloadArtifactUrl(repoName, buildId, artifactName, project);
if (!string.IsNullOrEmpty(downloadUrl))
var downloadUrl = await getDownloadArtifactUrl(buildId, artifactName, project);
if (string.IsNullOrEmpty(downloadUrl))
{
throw new Exception(string.Format("Failed to get download url for artifact {0} in build {1} in project {2}", artifactName, buildId, project));
}

if(!string.IsNullOrEmpty(filePath))
{
if(!string.IsNullOrEmpty(filePath))
if (!filePath.StartsWith("/"))
{
if (!filePath.StartsWith("/"))
{
filePath = "/" + filePath;
}
downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath;
filePath = "/" + filePath;
}

var downloadResp = await GetFromDevopsAsync(downloadUrl);
downloadResp.EnsureSuccessStatusCode();
return await downloadResp.Content.ReadAsStreamAsync();
downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath;
}
return null;

HttpResponseMessage downloadResp = await GetFromDevopsAsync(downloadUrl);
downloadResp.EnsureSuccessStatusCode();
return await downloadResp.Content.ReadAsStreamAsync();
}

private async Task<HttpResponseMessage> GetFromDevopsAsync(string request)
private async Task<string> getDownloadArtifactUrl(string buildId, string artifactName, string project)
{
var downloadResp = await _devopsClient.GetAsync(request);


if (!downloadResp.IsSuccessStatusCode)
{
var retryAfter = downloadResp.Headers.GetValues("Retry-After");
var rateLimitResource = downloadResp.Headers.GetValues("X-RateLimit-Resource");
var rateLimitDelay = downloadResp.Headers.GetValues("X-RateLimit-Delay");
var rateLimitLimit = downloadResp.Headers .GetValues("X-RateLimit-Limit");
var rateLimitRemaining = downloadResp.Headers.GetValues("X-RateLimit-Remaining");
var rateLimitReset = downloadResp.Headers.GetValues("X-RateLimit-Reset");

var traceMessage = $"request: {request} failed with statusCode: {downloadResp.StatusCode}," +
$"retryAfter: {retryAfter.FirstOrDefault()}, rateLimitResource: {rateLimitResource.FirstOrDefault()}, rateLimitDelay: {rateLimitDelay.FirstOrDefault()}," +
$"rateLimitLimit: {rateLimitLimit.FirstOrDefault()}, rateLimitRemaining: {rateLimitRemaining.FirstOrDefault()}, rateLimitReset: {rateLimitReset.FirstOrDefault()}";

_telemetryClient.TrackTrace(traceMessage);
}
var connection = await CreateVssConnection();
var buildClient = connection.GetClient<BuildHttpClient>();
var artifact = await buildClient.GetArtifactAsync(project, int.Parse(buildId), artifactName);
return artifact?.Resource?.DownloadUrl;
}

int count = 0;
int[] waitTimes = new int[] { 0, 1, 2, 4, 8, 16, 32, 64, 128, 256 };
while ((downloadResp.StatusCode == HttpStatusCode.TooManyRequests || downloadResp.StatusCode == HttpStatusCode.BadRequest) && count < waitTimes.Length)
{
_telemetryClient.TrackTrace($"Download request from devops artifact is either throttled or flaky, waiting {waitTimes[count]} seconds before retrying, Retry count: {count}");
await Task.Delay(TimeSpan.FromSeconds(waitTimes[count]));
downloadResp = await _devopsClient.GetAsync(request);
count++;
}
return downloadResp;
private async Task<VssConnection> CreateVssConnection()
{
var accessToken = await getAccessToken();
var token = new VssAadToken("Bearer", accessToken);
return new VssConnection(new Uri("https://dev.azure.com/azure-sdk/"), new VssAadCredential(token));
}

private async Task<string> GetDownloadArtifactUrl(string repoName, string buildId, string artifactName, string project)
private async Task<string> getAccessToken()
{
var artifactGetReq = GetArtifactRestAPIForRepo(repoName).Replace("{buildId}", buildId).Replace("{artifactName}", artifactName).Replace("{project}", project);
var response = await GetFromDevopsAsync(artifactGetReq);
response.EnsureSuccessStatusCode();
var buildResource = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
if (buildResource == null)
{
return null;
}
return buildResource.RootElement.GetProperty("resource").GetProperty("downloadUrl").GetString();
// APIView deployed instances uses managed identity to authenticate requests to Azure DevOps.
// For local testing, VS will use developer credentials to create token
var credential = new DefaultAzureCredential();
var tokenRequestContext = new TokenRequestContext(VssAadSettings.DefaultScopes);
var token = await credential.GetTokenAsync(tokenRequestContext, CancellationToken.None);
return token.Token;
}

private string GetArtifactRestAPIForRepo(string repoName)
private async Task<HttpResponseMessage> GetFromDevopsAsync(string request)
{
var downloadArtifactRestApi = _configuration["download-artifact-rest-api-for-" + repoName];
if (downloadArtifactRestApi == null)
var httpClient = new HttpClient();
var accessToken = await getAccessToken();
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
var maxRetryAttempts = 10;
var pauseBetweenFailures = TimeSpan.FromSeconds(2);

var retryPolicy = Policy
.Handle<HttpRequestException>()
.WaitAndRetryAsync(maxRetryAttempts, i => pauseBetweenFailures);

HttpResponseMessage downloadResp = null;
await retryPolicy.ExecuteAsync(async () =>
{
downloadArtifactRestApi = _configuration["download-artifact-rest-api"];
}
return downloadArtifactRestApi;
downloadResp = await httpClient.GetAsync(request);
});
return downloadResp;
}

public async Task RunPipeline(string pipelineName, string reviewDetails, string originalStorageUrl)
{
//Create dictionary of all required parametes to run tools - generate-<language>-apireview pipeline in azure devops
var reviewDetailsDict = new Dictionary<string, string> { { "Reviews", reviewDetails }, { "APIViewUrl", _hostUrl }, { "StorageContainerUrl", originalStorageUrl } };
var devOpsCreds = new VssBasicCredential("nobody", _configuration["Azure-Devops-PAT"]);
var devOpsConnection = new VssConnection(new Uri($"https://dev.azure.com/azure-sdk/"), devOpsCreds);
var devOpsConnection = await CreateVssConnection();
string projectName = _configuration["Azure-Devops-internal-project"] ?? "internal";

BuildHttpClient buildClient = await devOpsConnection.GetClientAsync<BuildHttpClient>();
var projectClient = await devOpsConnection.GetClientAsync<ProjectHttpClient>();
string envName = _configuration["apiview-deployment-environment"];
string updatedPipelineName = string.IsNullOrEmpty(envName) ? pipelineName : $"{pipelineName}-{envName}";
int definitionId = await GetPipelineId(updatedPipelineName, buildClient, projectName);
if (definitionId == 0)
{
throw new Exception(string.Format("Azure Devops pipeline is not found with name {0}. Please recheck and ensure pipeline exists with this name", updatedPipelineName));
if (definitionId == 0)
{
throw new Exception(string.Format("Azure Devops pipeline is not found with name {0}. Please recheck and ensure pipeline exists with this name", updatedPipelineName));
}

var definition = await buildClient.GetDefinitionAsync(projectName, definitionId);
var definition = await buildClient.GetDefinitionAsync(projectName, definitionId);
var project = await projectClient.GetProject(projectName);
await buildClient.QueueBuildAsync(new Build()
{
Definition = definition,
Project = project,
Parameters = JsonConvert.SerializeObject(reviewDetailsDict)
await buildClient.QueueBuildAsync(new Build()
{
Definition = definition,
Project = project,
Parameters = JsonConvert.SerializeObject(reviewDetailsDict)
});
}

private async Task<int> GetPipelineId(string pipelineName, BuildHttpClient client, string projectName)
{
var pipelines = await client.GetFullDefinitionsAsync2(project: projectName);
if (pipelines != null)
{
return pipelines.Single(p => p.Name == pipelineName).Id;
}
return 0;
private async Task<int> GetPipelineId(string pipelineName, BuildHttpClient client, string projectName)
{
var pipelines = await client.GetFullDefinitionsAsync2(project: projectName);
if (pipelines != null)
{
return pipelines.Single(p => p.Name == pipelineName).Id;
}
return 0;
}
}
}