diff --git a/releasenotes/v1.8.1.md b/releasenotes/v1.8.1.md index 3485c7b1c..834f3442b 100644 --- a/releasenotes/v1.8.1.md +++ b/releasenotes/v1.8.1.md @@ -1,2 +1,2 @@ Add `--use-github-storage` to `gh [gei|bbs2gh] migrate-repo` command to support uploading to a GitHub owned storage -Add `--use-github-storage` to `gh gei generate-script` command to support uploading to a GitHub owned storage \ No newline at end of file +Add `--use-github-storage` to `gh [gei|bbs2gh] generate-script` command to support uploading to a GitHub owned storage diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index de22e99a8..dc54136d0 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -14,7 +14,6 @@ namespace OctoshiftCLI.IntegrationTests; [Collection("Integration Tests")] public sealed class BbsToGithub : IDisposable { - private const string SSH_KEY_FILE = "ssh_key.pem"; private const string AWS_REGION = "us-east-1"; @@ -33,6 +32,8 @@ public sealed class BbsToGithub : IDisposable private readonly DateTime _startTime; private readonly string _azureStorageConnectionString; + public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage } + public BbsToGithub(ITestOutputHelper output) { _startTime = DateTime.Now; @@ -67,10 +68,11 @@ public BbsToGithub(ITestOutputHelper output) } [Theory] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true)] - [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true)] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false)] - public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload) + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)] + [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)] + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)] + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)] + public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption) { var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}"; @@ -110,17 +112,21 @@ await retryPolicy.Retry(async () => } var archiveUploadOptions = ""; - if (useAzureForArchiveUpload) + if (uploadOption == ArchiveUploadOption.AzureStorage) { _tokens.Add("AZURE_STORAGE_CONNECTION_STRING", _azureStorageConnectionString); } - else + else if (uploadOption == ArchiveUploadOption.AwsS3) { _tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID")); _tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY")); var awsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME"); archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION}"; } + else if (uploadOption == ArchiveUploadOption.GithubStorage) + { + archiveUploadOptions = " --use-github-storage"; + } await _targetHelper.RunBbsCliMigration( $"generate-script --github-org {githubTargetOrg} --bbs-server-url {bbsServer} --bbs-project {bbsProjectKey}{archiveDownloadOptions}{archiveUploadOptions}", _tokens); diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs index 2064072d0..91d402928 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -34,4 +34,32 @@ public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning() _mockOctoLogger.Verify(x => x.LogWarning(It.Is(x => x.ToLower().Contains("--ssh-port is set to 7999")))); } + + [Fact] + public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsBucketName = "my-bucket"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + [Fact] + public void It_Throws_If_Both_AwsRegion_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsRegion = "aws-region"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } } diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index 211306a5b..6394f51c7 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -2,7 +2,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using FluentAssertions; using Moq; using OctoshiftCLI.BbsToGithub.Commands.GenerateScript; using OctoshiftCLI.Contracts; @@ -261,6 +260,51 @@ exit 1 _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); } + [Fact] + public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_And_AWS_Not_Validated_When_UseGithubStorage() + { + // Arrange + _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + BbsServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + UseGithubStorage = true + }; + await _handler.Handle(args); + + var expectedAws = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +} +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + + var expectedAzure = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAws, 0, 0))))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAzure, 0, 0))))); + } + [Fact] public async Task Validates_Env_Vars_SMB_PASSWORD() { @@ -701,6 +745,48 @@ public async Task One_Repo_With_Aws_Bucket_Name_And_Region() _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); } + [Fact] + public async Task BBS_Single_Repo_With_UseGithubStorage() + { + // Arrange + var TARGET_API_URL = "https://foo.com/api/v3"; + const string BBS_PROJECT_KEY = "BBS-PROJECT"; + const string BBS_REPO_SLUG = "repo-slug"; + + _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"), + }); + _mockBbsApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), + }); + + + // Act + var args = new GenerateScriptCommandArgs + { + BbsServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo("unit-test-output"), + UseGithubStorage = true, + TargetApiUrl = TARGET_API_URL, + BbsProject = BBS_PROJECT_KEY, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => + script.Contains("--bbs-server-url \"http://bbs-server-url\"") && + script.Contains("--bbs-project \"BBS-PROJECT\"") && + script.Contains("--github-org \"GITHUB-ORG\"") && + script.Contains("--use-github-storage") +))); + + } + + + private string TrimNonExecutableLines(string script, int skipFirst = 9, int skipLast = 0) { var lines = script.Split(new[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries).AsEnumerable(); diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs index db247883e..284822db0 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -36,7 +36,7 @@ public void Should_Have_Options() { _command.Should().NotBeNull(); _command.Name.Should().Be("generate-script"); - _command.Options.Count.Should().Be(20); + _command.Options.Count.Should().Be(21); TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true); TestHelpers.VerifyCommandOption(_command.Options, "github-org", true); @@ -58,6 +58,7 @@ public void Should_Have_Options() TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false); TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true); } [Fact] diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs index b5016f6af..5a09fcc84 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -35,6 +35,7 @@ public GenerateScriptCommand() : base( AddOption(AwsRegion); AddOption(KeepArchive); AddOption(NoSslVerify); + AddOption(UseGithubStorage); } public Option BbsServerUrl { get; } = new( @@ -124,6 +125,12 @@ public GenerateScriptCommand() : base( Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" }; + public Option UseGithubStorage { get; } = new("--use-github-storage") + { + IsHidden = true, + Description = "Enables multipart uploads to a GitHub owned storage for use during migration", + }; + public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp) { if (args is null) diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 2e2c6c93f..d54077566 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -27,6 +27,7 @@ public class GenerateScriptCommandArgs : CommandArgs public bool KeepArchive { get; set; } public bool NoSslVerify { get; set; } public string TargetApiUrl { get; set; } + public bool UseGithubStorage { get; set; } public override void Validate(OctoLogger log) { @@ -35,6 +36,16 @@ public override void Validate(OctoLogger log) throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); } + if (AwsBucketName.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + if (AwsRegion.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } + if (SshPort == 7999) { log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH."); diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 68be3956b..0656cc13f 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -71,7 +71,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); content.AppendLine(VALIDATE_AWS_SECRET_ACCESS_KEY); } - else + else if (!args.UseGithubStorage) { content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING); } @@ -135,9 +135,10 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb var noSslVerify = args.NoSslVerify ? " --no-ssl-verify" : ""; var targetRepoVisibility = " --target-repo-visibility private"; var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : ""; + var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; return $"gh bbs2gh migrate-repo{targetApiUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsSharedHomeOption}{bbsProjectOption}{bbsRepoOption}{sshArchiveDownloadOptions}" + - $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}"; + $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; } private string Exec(string script) => Wrap(script, "Exec");