Merge branch 'main' into friendly-error-when-missing-github-data-prov…

…ider

src/Octoshift/Services/GithubApi.cs

@@ -574,11 +574,10 @@ public virtual async Task<int> GetIdpGroupId(string org, string groupName)
     {
         var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups";
 
-        // TODO: Need to implement paging
-        var response = await _client.GetAsync(url);
-        var data = JObject.Parse(response);
+        var group = await _client.GetAllAsync(url, data => (JArray)data["groups"])
+            .SingleAsync(x => string.Equals((string)x["group_name"], groupName, StringComparison.OrdinalIgnoreCase));
 
-        return (int)data["groups"].Children().Single(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper())["group_id"];
+        return (int)group["group_id"];
     }
 
     public virtual async Task<string> GetTeamSlug(string org, string teamName)

src/Octoshift/Services/GithubClient.cs

@@ -47,21 +47,32 @@ public GithubClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers
 
     public virtual async Task<string> GetAsync(string url, Dictionary<string, string> customHeaders = null) => (await GetWithRetry(url, customHeaders)).Content;
 
-    public virtual async IAsyncEnumerable<JToken> GetAllAsync(string url, Dictionary<string, string> customHeaders = null)
+    public virtual IAsyncEnumerable<JToken> GetAllAsync(string url, Dictionary<string, string> customHeaders = null) =>
+        GetAllAsync(url, jToken => (JArray)jToken, customHeaders);
+
+    public virtual async IAsyncEnumerable<JToken> GetAllAsync(
+        string url,
+        Func<JToken, JArray> resultCollectionSelector,
+        Dictionary<string, string> customHeaders = null)
     {
+        if (resultCollectionSelector is null)
+        {
+            throw new ArgumentNullException(nameof(resultCollectionSelector));
+        }
+
         var nextUrl = url;
         do
         {
             var (content, headers) = await GetWithRetry(nextUrl, customHeaders: customHeaders);
-            foreach (var jToken in JArray.Parse(content))
+            var jContent = JToken.Parse(content);
+            foreach (var jToken in resultCollectionSelector(jContent))
             {
                 yield return jToken;
             }
 
             nextUrl = GetNextUrl(headers);
         } while (nextUrl != null);
     }
-
     public virtual async Task<string> PostAsync(string url, object body, Dictionary<string, string> customHeaders = null) =>
         (await SendAsync(HttpMethod.Post, url, body, customHeaders: customHeaders)).Content;
 

src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs

@@ -15,7 +15,7 @@ public AdoBasicToGithub(ITestOutputHelper output) : base(output)
         public async Task Basic()
         {
             var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}";
-            var githubOrg = $"e2e-testing-ado-basic-{TestHelper.GetOsName()}";
+            var githubOrg = $"octoshift-e2e-ado-basic-{TestHelper.GetOsName()}-2";
             var teamProject1 = "gei-e2e-1";
             var teamProject2 = "gei-e2e-2";
             var adoRepo1 = teamProject1;

src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs

@@ -15,7 +15,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output)
         public async Task With_Inventory_Report_Csv()
         {
             var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}";
-            var githubOrg = $"e2e-testing-ado-csv-{TestHelper.GetOsName()}";
+            var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}";
             var teamProject1 = "gei-e2e-1";
             var teamProject2 = "gei-e2e-2";
             var adoRepo1 = teamProject1;

src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs

@@ -17,7 +17,7 @@ public AdoServerToGithub(ITestOutputHelper output) : base(output, ADO_SERVER_URL
         public async Task Basic()
         {
             var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}";
-            var githubOrg = $"e2e-testing-ado-server-{TestHelper.GetOsName()}";
+            var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}";
             var teamProject1 = "gei-e2e-1";
             var teamProject2 = "gei-e2e-2";
             var adoRepo1 = teamProject1;

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -71,9 +71,11 @@ public BbsToGithub(ITestOutputHelper output)
     public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload)
     {
         var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}";
-        var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}";
+        var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}";
         var repo1 = $"{bbsProjectKey}-repo-1";
         var repo2 = $"{bbsProjectKey}-repo-2";
+        var targetRepo1 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-1";
+        var targetRepo2 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-2";
 
         var sourceBbsApi = new BbsApi(_sourceBbsClient, bbsServer, _logger);
         var sourceHelper = new TestHelper(_output, sourceBbsApi, _sourceBbsClient, bbsServer);
@@ -87,10 +89,10 @@ await retryPolicy.Retry(async () =>
             await _targetHelper.ResetGithubTestEnvironment(githubTargetOrg);
 
             await sourceHelper.CreateBbsProject(bbsProjectKey);
-            await sourceHelper.CreateBbsRepo(bbsProjectKey, "repo-1");
-            await sourceHelper.InitializeBbsRepo(bbsProjectKey, "repo-1");
-            await sourceHelper.CreateBbsRepo(bbsProjectKey, "repo-2");
-            await sourceHelper.InitializeBbsRepo(bbsProjectKey, "repo-2");
+            await sourceHelper.CreateBbsRepo(bbsProjectKey, repo1);
+            await sourceHelper.InitializeBbsRepo(bbsProjectKey, repo1);
+            await sourceHelper.CreateBbsRepo(bbsProjectKey, repo2);
+            await sourceHelper.InitializeBbsRepo(bbsProjectKey, repo2);
         });
 
         var archiveDownloadOptions = $" --ssh-user octoshift --ssh-private-key {SSH_KEY_FILE}";
@@ -123,10 +125,10 @@ await _targetHelper.RunBbsCliMigration(
 
         _targetHelper.AssertNoErrorInLogs(_startTime);
 
-        await _targetHelper.AssertGithubRepoExists(githubTargetOrg, repo1);
-        await _targetHelper.AssertGithubRepoExists(githubTargetOrg, repo2);
-        await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, repo1);
-        await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, repo2);
+        await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo1);
+        await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo2);
+        await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo1);
+        await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo2);
 
         // TODO: Assert migration logs are downloaded
     }

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -65,7 +65,7 @@ public GhesToGithub(ITestOutputHelper output)
     public async Task Basic()
     {
         var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}";
-        var githubTargetOrg = $"e2e-testing-ghes-{TestHelper.GetOsName()}";
+        var githubTargetOrg = $"octoshift-e2e-ghes-{TestHelper.GetOsName()}";
         const string repo1 = "repo-1";
         const string repo2 = "repo-2";
 

src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs

@@ -43,8 +43,8 @@ public GithubToGithub(ITestOutputHelper output)
         [Fact]
         public async Task Basic()
         {
-            var githubSourceOrg = $"e2e-testing-source-{TestHelper.GetOsName()}";
-            var githubTargetOrg = $"e2e-testing-ghec-{TestHelper.GetOsName()}";
+            var githubSourceOrg = $"octoshift-e2e-source-{TestHelper.GetOsName()}";
+            var githubTargetOrg = $"octoshift-e2e-ghec-{TestHelper.GetOsName()}";
             var repo1 = "repo-1";
             var repo2 = "repo-2";
 

src/OctoshiftCLI.IntegrationTests/TestHelper.cs

@@ -487,8 +487,10 @@ private async Task CreateRepo(string org, string repo, bool isPrivate, bool isIn
 
         private async Task<IEnumerable<string>> GetRepoCommitShas(string org, string repo)
         {
+            var retryPolicy = new RetryPolicy(null);
+
             var url = $"{GithubApiBaseUrl}/repos/{org}/{repo}/commits";
-            var commits = await _githubClient.GetAllAsync(url).ToListAsync();
+            var commits = await retryPolicy.Retry(async () => await _githubClient.GetAllAsync(url).ToListAsync());
             return commits.Select(x => (string)x["sha"]).ToList();
         }
 

src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs

@@ -1606,25 +1606,27 @@ public async Task GetIdpGroupId_Returns_The_Idp_Group_Id()
 
         var url = $"https://api.github.com/orgs/{GITHUB_ORG}/external-groups";
         const int expectedGroupId = 123;
-        var response = $@"
-            {{
-                ""groups"": [
-                    {{
-                       ""group_id"": ""{expectedGroupId}"",
-                       ""group_name"": ""{groupName}"",
-                       ""updated_at"": ""2021-01-24T11:31:04-06:00""
-                    }},
-                    {{
-                       ""group_id"": ""456"",
-                       ""group_name"": ""Octocat admins"",
-                       ""updated_at"": ""2021-03-24T11:31:04-06:00""
-                    }},
-                ]
-            }}";
+
+        var group1 = new
+        {
+            group_id = expectedGroupId,
+            group_name = groupName,
+            updated_at = DateTime.Parse("2021-01-24T11:31:04-06:00")
+        };
+        var group2 = new
+        {
+            group_id = "456",
+            group_name = "Octocat admins",
+            updated_at = DateTime.Parse("2021-03-24T11:31:04-06:00")
+        };
 
         _githubClientMock
-            .Setup(m => m.GetAsync(url, null))
-            .ReturnsAsync(response);
+            .Setup(m => m.GetAllAsync(url, It.IsAny<Func<JToken, JArray>>(), null))
+            .Returns(new[]
+            {
+                JToken.FromObject(group1),
+                JToken.FromObject(group2)
+            }.ToAsyncEnumerable());
 
         // Act
         var actualGroupId = await _githubApi.GetIdpGroupId(GITHUB_ORG, groupName);

src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs

@@ -1351,6 +1351,48 @@ public async Task GetAllAsync_Retries_On_Non_Success()
         results.Should().BeEquivalentTo(firstItem, secondItem);
     }
 
+    [Fact]
+    public async Task GetAllAsync_Should_Use_Result_Collection_Selector()
+    {
+        // Arrange
+        const string url = "https://api.github.com/orgs/foo/external-groups";
+
+        const string firstGroupId = "123";
+        const string firstGroupName = "Octocat readers";
+        const string secondGroupId = "456";
+        const string secondGroupName = "Octocat admins";
+        const string response = $@"
+            {{
+                ""groups"": [
+                    {{
+                       ""group_id"": ""{firstGroupId}"",
+                       ""group_name"": ""{firstGroupName}"",
+                       ""updated_at"": ""2021-01-24T11:31:04-06:00""
+                    }},
+                    {{
+                       ""group_id"": ""{secondGroupId}"",
+                       ""group_name"": ""{secondGroupName}"",
+                       ""updated_at"": ""2021-03-24T11:31:04-06:00""
+                    }},
+                ]
+            }}";
+
+        var handlerMock = MockHttpHandler(req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, CreateHttpResponseFactory(content: response));
+
+        using var httpClient = new HttpClient(handlerMock.Object);
+        var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN);
+
+        // Act
+        var results = await githubClient.GetAllAsync(url, data => (JArray)data["groups"]).ToListAsync();
+
+        // Assert
+        results.Should().HaveCount(2);
+        results[0]["group_id"].Value<string>().Should().Be(firstGroupId);
+        results[0]["group_name"].Value<string>().Should().Be(firstGroupName);
+        results[1]["group_id"].Value<string>().Should().Be(secondGroupId);
+        results[1]["group_name"].Value<string>().Should().Be(secondGroupName);
+    }
+
     [Fact]
     public async Task PostGraphQLWithPaginationAsync_Should_Return_All_Pages()
     {