-
Notifications
You must be signed in to change notification settings - Fork 83
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
FINAL: Git storage option feature #1279
base: main
Are you sure you want to change the base?
Changes from all commits
781b6de
5d36e96
aab0557
a8797ca
95844ea
1a5e198
420db29
ffc3e21
1fca83d
f81e02a
2c8de4a
619e6da
bf5eef2
60e9ed1
cd88309
c161de8
cd6311a
d4e2344
4becd72
abd757c
adac212
a121256
5f67714
f7ffcef
d9e9149
c959ccc
d9a3d0a
08fc506
ffaf1e8
c8b1d3b
a2784b6
5a61216
947fcab
ec9011c
f66f34a
230503f
61726e2
7d85f23
b03be62
006e188
da59731
ca6c476
121319c
ffc219e
85190e8
2894bb8
abbbaaf
981cd15
e538d00
2d0a63d
0821922
ad111f0
550f4b8
2d06def
fe78726
0e532f1
18b3c34
e97303b
54d09b2
ebc9f65
92633b0
67f39b1
80d5355
460a525
015ed9c
ae4b940
0099104
64223bb
dbe7db2
03b7266
fead8a7
1d65610
3f04d2c
2b241a6
e84134e
fb3ccd4
6a24f7a
99cc3fc
e2585dc
6e58b5f
3b3a301
e1eabc5
e3fff28
8fce586
60d6f83
c949795
ffaf286
cbc0d1b
a7eae8b
099cfda
066bd38
502d7a2
8d594a3
156cc07
4b4d605
0e8093b
6533c7a
090547a
0cba519
c93e44d
ce20d20
ede2cd8
958007b
c37a19c
2bade7a
eae73c0
e50309b
451ed8d
8167f2f
47dbcb4
0b3948d
1478850
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,162 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.IO; | ||
using System.Linq; | ||
using System.Net.Http; | ||
using System.Threading.Tasks; | ||
using System.Web; | ||
using Newtonsoft.Json.Linq; | ||
using OctoshiftCLI.Extensions; | ||
|
||
namespace OctoshiftCLI.Services; | ||
|
||
public class ArchiveUploader | ||
{ | ||
private readonly GithubClient _client; | ||
private readonly OctoLogger _log; | ||
internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB | ||
|
||
private const string BASE_URL = "https://uploads.github.com"; | ||
|
||
public ArchiveUploader(GithubClient client, OctoLogger log) | ||
{ | ||
_client = client; | ||
_log = log; | ||
} | ||
public virtual async Task<string> Upload(Stream archiveContent, string archiveName, string orgDatabaseId) | ||
{ | ||
if (archiveContent == null) | ||
{ | ||
throw new ArgumentNullException(nameof(archiveContent), "The archive content stream cannot be null."); | ||
} | ||
|
||
using var streamContent = new StreamContent(archiveContent); | ||
streamContent.Headers.ContentType = new("application/octet-stream"); | ||
|
||
var isMultipart = archiveContent.Length > _streamSizeLimit; // Determines if stream size is greater than 100MB | ||
|
||
string response; | ||
|
||
if (isMultipart) | ||
{ | ||
var url = $"{BASE_URL}/organizations/{orgDatabaseId.EscapeDataString()}/gei/archive/blobs/uploads"; | ||
|
||
response = await UploadMultipart(archiveContent, archiveName, url); | ||
return response; | ||
} | ||
else | ||
{ | ||
var url = $"{BASE_URL}/organizations/{orgDatabaseId.EscapeDataString()}/gei/archive?name={archiveName.EscapeDataString()}"; | ||
|
||
response = await _client.PostAsync(url, streamContent); | ||
var data = JObject.Parse(response); | ||
return (string)data["uri"]; | ||
} | ||
} | ||
|
||
private async Task<string> UploadMultipart(Stream archiveContent, string archiveName, string uploadUrl) | ||
{ | ||
var buffer = new byte[_streamSizeLimit]; | ||
|
||
try | ||
{ | ||
// 1. Start the upload | ||
var startHeaders = await StartUpload(uploadUrl, archiveName, archiveContent.Length); | ||
|
||
var nextUrl = GetNextUrl(startHeaders); | ||
|
||
var guid = HttpUtility.ParseQueryString(nextUrl.Query)["guid"]; | ||
|
||
// 2. Upload parts | ||
int bytesRead; | ||
var partsRead = 0; | ||
var totalParts = (long)Math.Ceiling((double)archiveContent.Length / _streamSizeLimit); | ||
while ((bytesRead = await archiveContent.ReadAsync(buffer)) > 0) | ||
{ | ||
nextUrl = await UploadPart(buffer, bytesRead, nextUrl.ToString(), partsRead, totalParts); | ||
partsRead++; | ||
} | ||
|
||
// 3. Complete the upload | ||
await CompleteUpload(nextUrl.ToString()); | ||
|
||
return $"gei://archive/{guid}"; | ||
} | ||
catch (Exception ex) | ||
{ | ||
throw new OctoshiftCliException("Failed during multipart upload.", ex); | ||
} | ||
} | ||
|
||
private async Task<IEnumerable<KeyValuePair<string, IEnumerable<string>>>> StartUpload(string uploadUrl, string archiveName, long contentSize) | ||
{ | ||
_log.LogInformation($"Starting archive upload into GitHub owned storage: {archiveName}..."); | ||
|
||
var body = new | ||
{ | ||
content_type = "application/octet-stream", | ||
name = archiveName, | ||
size = contentSize | ||
}; | ||
|
||
try | ||
{ | ||
var (responseContent, headers) = await _client.PostWithFullResponseAsync(uploadUrl, body); | ||
Check warning Code scanning / CodeQL Useless assignment to local variable Warning
This assignment to
responseContent Error loading related location Loading |
||
return headers.ToList(); | ||
} | ||
catch (Exception ex) | ||
{ | ||
throw new OctoshiftCliException("Failed to start upload.", ex); | ||
} | ||
Comment on lines
+107
to
+110
Check notice Code scanning / CodeQL Generic catch clause Note
Generic catch clause.
|
||
} | ||
|
||
private async Task<Uri> UploadPart(byte[] body, int bytesRead, string nextUrl, int partsRead, long totalParts) | ||
{ | ||
_log.LogInformation($"Uploading part {partsRead + 1}/{totalParts}..."); | ||
using var content = new ByteArrayContent(body, 0, bytesRead); | ||
content.Headers.ContentType = new("application/octet-stream"); | ||
|
||
try | ||
{ | ||
// Make the PATCH request and retrieve headers | ||
var (responseContent, headers) = await _client.PatchWithFullResponseAsync(nextUrl, content); | ||
Check warning Code scanning / CodeQL Useless assignment to local variable Warning
This assignment to
responseContent Error loading related location Loading |
||
|
||
// Retrieve the next URL from the response headers | ||
return GetNextUrl(headers.ToList()); | ||
} | ||
catch (Exception ex) | ||
{ | ||
throw new OctoshiftCliException("Failed to upload part.", ex); | ||
} | ||
Comment on lines
+127
to
+130
Check notice Code scanning / CodeQL Generic catch clause Note
Generic catch clause.
|
||
} | ||
|
||
private async Task CompleteUpload(string lastUrl) | ||
{ | ||
try | ||
{ | ||
await _client.PutAsync(lastUrl, ""); | ||
_log.LogInformation("Finished uploading archive"); | ||
} | ||
catch (Exception ex) | ||
{ | ||
throw new OctoshiftCliException("Failed to complete upload.", ex); | ||
} | ||
Comment on lines
+140
to
+143
Check notice Code scanning / CodeQL Generic catch clause Note
Generic catch clause.
|
||
} | ||
|
||
private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers) | ||
{ | ||
// Use FirstOrDefault to safely handle missing Location headers | ||
var locationHeader = headers.First(header => header.Key.Equals("Location", StringComparison.OrdinalIgnoreCase)); | ||
|
||
if (!string.IsNullOrEmpty(locationHeader.Key)) | ||
{ | ||
var locationValue = locationHeader.Value.FirstOrDefault(); | ||
if (locationValue.HasValue()) | ||
{ | ||
var fullUrl = $"{BASE_URL}{locationValue}"; | ||
return new Uri(fullUrl); | ||
} | ||
} | ||
throw new OctoshiftCliException("Location header is missing in the response, unable to retrieve next URL for multipart upload."); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,6 +28,7 @@ public sealed class BbsToGithub : IDisposable | |
private readonly HttpClient _sourceBbsHttpClient; | ||
private readonly BbsClient _sourceBbsClient; | ||
private readonly BlobServiceClient _blobServiceClient; | ||
private readonly ArchiveUploader _archiveUploader; | ||
private readonly Dictionary<string, string> _tokens; | ||
private readonly DateTime _startTime; | ||
private readonly string _azureStorageConnectionString; | ||
|
@@ -57,18 +58,20 @@ public BbsToGithub(ITestOutputHelper output) | |
|
||
_targetGithubHttpClient = new HttpClient(); | ||
_targetGithubClient = new GithubClient(_logger, _targetGithubHttpClient, new VersionChecker(_versionClient, _logger), new RetryPolicy(_logger), new DateTimeProvider(), targetGithubToken); | ||
_targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger)); | ||
_archiveUploader = new ArchiveUploader(_targetGithubClient, _logger); | ||
_targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger), _archiveUploader); | ||
|
||
_blobServiceClient = new BlobServiceClient(_azureStorageConnectionString); | ||
|
||
_targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient, _blobServiceClient); | ||
} | ||
|
||
[Theory] | ||
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true)] | ||
[InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true)] | ||
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false)] | ||
public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload) | ||
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true, false)] | ||
[InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true, false)] | ||
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false, false)] | ||
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", false, false, true)] | ||
public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload, bool useGithubStorage) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. At this point I recommend adding an enum for the upload options as a class member (nested enum) right after all public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage }; Then change the method signature and the inline data like the following: [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)]
[InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)]
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)]
[InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)]
public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload, bool useGithubStorage) |
||
{ | ||
var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; | ||
var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}"; | ||
|
@@ -117,7 +120,7 @@ await retryPolicy.Retry(async () => | |
_tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID")); | ||
_tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY")); | ||
var awsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME"); | ||
archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION}"; | ||
archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION} --use-github-storage {useGithubStorage}"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There are a couple of things here:
|
||
} | ||
|
||
await _targetHelper.RunBbsCliMigration( | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -27,6 +27,7 @@ public sealed class GhesToGithub : IDisposable | |
private readonly BlobServiceClient _blobServiceClient; | ||
private readonly Dictionary<string, string> _tokens; | ||
private readonly DateTime _startTime; | ||
private readonly ArchiveUploader _archiveUploader; | ||
|
||
public GhesToGithub(ITestOutputHelper output) | ||
{ | ||
|
@@ -46,23 +47,26 @@ public GhesToGithub(ITestOutputHelper output) | |
}; | ||
|
||
_versionClient = new HttpClient(); | ||
_archiveUploader = new ArchiveUploader(_targetGithubClient, logger); | ||
|
||
_sourceGithubHttpClient = new HttpClient(); | ||
_sourceGithubClient = new GithubClient(logger, _sourceGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger), new DateTimeProvider(), sourceGithubToken); | ||
_sourceGithubApi = new GithubApi(_sourceGithubClient, GHES_API_URL, new RetryPolicy(logger)); | ||
_sourceGithubApi = new GithubApi(_sourceGithubClient, GHES_API_URL, new RetryPolicy(logger), _archiveUploader); | ||
|
||
_targetGithubHttpClient = new HttpClient(); | ||
_targetGithubClient = new GithubClient(logger, _targetGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger), new DateTimeProvider(), targetGithubToken); | ||
_targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(logger)); | ||
_targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(logger), _archiveUploader); | ||
|
||
_blobServiceClient = new BlobServiceClient(azureStorageConnectionString); | ||
|
||
_sourceHelper = new TestHelper(_output, _sourceGithubApi, _sourceGithubClient) { GithubApiBaseUrl = GHES_API_URL }; | ||
_targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient, _blobServiceClient); | ||
} | ||
|
||
[Fact] | ||
public async Task Basic() | ||
[Theory] | ||
[InlineData(false)] | ||
[InlineData(true)] | ||
public async Task Basic(bool useGithubStorage) | ||
{ | ||
var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}"; | ||
var githubTargetOrg = $"octoshift-e2e-ghes-{TestHelper.GetOsName()}"; | ||
|
@@ -83,7 +87,7 @@ await retryPolicy.Retry(async () => | |
}); | ||
|
||
await _targetHelper.RunGeiCliMigration( | ||
$"generate-script --github-source-org {githubSourceOrg} --github-target-org {githubTargetOrg} --ghes-api-url {GHES_API_URL} --download-migration-logs", _tokens); | ||
$"generate-script --github-source-org {githubSourceOrg} --github-target-org {githubTargetOrg} --ghes-api-url {GHES_API_URL} --use-github-storage {useGithubStorage} --download-migration-logs", _tokens); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In case of |
||
|
||
_targetHelper.AssertNoErrorInLogs(_startTime); | ||
|
||
|
@@ -95,7 +99,6 @@ await _targetHelper.RunGeiCliMigration( | |
_targetHelper.AssertMigrationLogFileExists(githubTargetOrg, repo1); | ||
_targetHelper.AssertMigrationLogFileExists(githubTargetOrg, repo2); | ||
} | ||
|
||
public void Dispose() | ||
{ | ||
_sourceGithubHttpClient?.Dispose(); | ||
|
Check notice
Code scanning / CodeQL
Generic catch clause Note