FIX JENKINS-63048 (#31)

* FIX JENKINS-63048

use the ref=next link in the response header to support paged requests.

* fixed escape sequence and GiteaConnectionMock

* added https://repo.jenkins-ci.org/public repository to pom

* moved repository configuration to end of file

* added unit tests for paged requests

Co-authored-by: Julian Liebert <julian.liebert@seitenbau.com>

pom.xml

@@ -115,4 +115,17 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
+
+    <repositories>  
+        <repository>
+            <id>repo.jenkins-ci.org</id>
+            <url>https://repo.jenkins-ci.org/public/</url>
+        </repository>
+    </repositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>repo.jenkins-ci.org</id>
+            <url>https://repo.jenkins-ci.org/public/</url>
+        </pluginRepository>
+    </pluginRepositories>
 </project>

src/main/java/org/jenkinsci/plugin/gitea/client/impl/DefaultGiteaConnection.java

@@ -45,7 +45,11 @@
 import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import javax.net.ssl.HttpsURLConnection;
 import jenkins.model.Jenkins;
 import org.apache.commons.codec.binary.Base64;
@@ -959,20 +963,35 @@ private <T> T patch(UriTemplate template, Object body, final Class<T> modelClass
         }
     }
 
+    private Pattern nextPagePattern = Pattern.compile("<(.*)>;\\s*rel=\"next\"");
+
     private <T> List<T> getList(UriTemplate template, final Class<T> modelClass)
             throws IOException, InterruptedException {
-        HttpURLConnection connection = openConnection(template);
+        return getList(template.expand(), modelClass);
+    }
+
+    private <T> List<T> getList(String url, final Class<T> modelClass) throws IOException, InterruptedException {
+        HttpURLConnection connection = openConnection(url);
         withAuthentication(connection);
         try {
             connection.connect();
             int status = connection.getResponseCode();
+
             if (status / 100 == 2) {
+                Optional<String> next = Optional.ofNullable(connection.getHeaderField("Link"))
+                        .map(nextPagePattern::matcher)
+                        .filter(Matcher::find)
+                        .map(matcher -> matcher.group(1));
+
                 try (InputStream is = connection.getInputStream()) {
-                    List<T> list = mapper.readerFor(mapper.getTypeFactory()
-                            .constructCollectionType(List.class, modelClass))
+                    List<T> list = mapper
+                            .readerFor(mapper.getTypeFactory().constructCollectionType(List.class, modelClass))
                             .readValue(is);
+                    if (next.isPresent()) {
+                        list.addAll(getList(next.get(), modelClass));
+                    }
                     // strip null values from the list
-                    for (Iterator<T> iterator = list.iterator(); iterator.hasNext(); ) {
+                    for (Iterator<T> iterator = list.iterator(); iterator.hasNext();) {
                         if (iterator.next() == null) {
                             iterator.remove();
                         }
@@ -986,9 +1005,13 @@ private <T> List<T> getList(UriTemplate template, final Class<T> modelClass)
         }
     }
 
+    private HttpURLConnection openConnection(UriTemplate template) throws IOException {
+       return openConnection(template.expand());
+    }
+
     @Restricted(NoExternalUse.class)
-    protected HttpURLConnection openConnection(UriTemplate template) throws IOException {
-        URL url = new URL(template.expand());
+    protected HttpURLConnection openConnection(String spec) throws IOException {
+        URL url = new URL(spec);
         Jenkins jenkins = Jenkins.get();
         if (jenkins.proxy == null) {
             return (HttpURLConnection) url.openConnection();

src/test/java/org/jenkinsci/plugin/gitea/client/impl/DefaultGiteaConnection_PagedRequests_Test.java

@@ -0,0 +1,212 @@
+package org.jenkinsci.plugin.gitea.client.impl;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.util.HashMap;
+import java.util.List;
+
+import org.jenkinsci.plugin.gitea.client.api.GiteaAuthNone;
+import org.jenkinsci.plugin.gitea.client.api.GiteaBranch;
+import org.jenkinsci.plugin.gitea.client.api.GiteaCommitStatus;
+import org.jenkinsci.plugin.gitea.client.api.GiteaHook;
+import org.jenkinsci.plugin.gitea.client.api.GiteaIssue;
+import org.jenkinsci.plugin.gitea.client.api.GiteaOrganization;
+import org.jenkinsci.plugin.gitea.client.api.GiteaOwner;
+import org.jenkinsci.plugin.gitea.client.api.GiteaPullRequest;
+import org.jenkinsci.plugin.gitea.client.api.GiteaRepository;
+import org.jenkinsci.plugin.gitea.client.api.GiteaTag;
+import org.jenkinsci.plugin.gitea.client.api.GiteaUser;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class DefaultGiteaConnection_PagedRequests_Test {
+
+    private GiteaRepository giteaRepository;
+
+    @Before
+    public void reset() {
+        giteaRepository = new GiteaRepository(
+                new GiteaOwner("", "", "", ""),
+                null, "", "", "",
+                true, false, false, false,
+                "", "", "", "",
+                0L, 0L, 0L, 0L, "",
+                null
+        );
+    }
+
+    @Test
+    public void test_fetchOrganizationRepositories_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/orgs//repos";
+        String page2Url = "http://server.com/api/v1/orgs//repos?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "repoResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "repoResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaRepository> repositories = giteaConnection
+                    .fetchRepositories(new GiteaOrganization("", "", "", "", "", ""));
+            assertThat(repositories.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchUserRepositories_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/users//repos";
+        String page2Url = "http://server.com/api/v1/users//repos?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "repoResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "repoResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaRepository> repositories = giteaConnection.fetchRepositories(new GiteaOwner("", "", "", ""));
+            assertThat(repositories.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchCurrentUserRepositories_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/user/repos";
+        String page2Url = "http://server.com/api/v1/user/repos?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "repoResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "repoResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaRepository> repositories = giteaConnection.fetchCurrentUserRepositories();
+            assertThat(repositories.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchBranches_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///branches";
+        String page2Url = "http://server.com/api/v1/repos///branches?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "branchesResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "branchesResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaBranch> branches = giteaConnection.fetchBranches("", "");
+            assertThat(branches.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchTags_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///tags";
+        String page2Url = "http://server.com/api/v1/repos///tags?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "tagsResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "tagsResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaTag> tags = giteaConnection.fetchTags("", "");
+            assertThat(tags.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchCollaborators_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///collaborators";
+        String page2Url = "http://server.com/api/v1/repos///collaborators?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "usersResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "usersResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaUser> users = giteaConnection.fetchCollaborators("", "");
+            assertThat(users.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchHooks_from_user_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///hooks";
+        String page2Url = "http://server.com/api/v1/repos///hooks?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "hooksResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "hooksResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaHook> hooks = giteaConnection.fetchHooks("", "");
+            assertThat(hooks.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchHooks_from_org_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/orgs//hooks";
+        String page2Url = "http://server.com/api/v1/orgs//hooks?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "hooksResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "hooksResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaHook> hooks = giteaConnection.fetchHooks("");
+            assertThat(hooks.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchCommitStatuses_from_org_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///statuses/sha";
+        String page2Url = "http://server.com/api/v1/repos///statuses/sha?page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "commitStatusResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "commitStatusResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaCommitStatus> commitStates = giteaConnection.fetchCommitStatuses(giteaRepository, "sha");
+            assertThat(commitStates.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchPullRequests_from_org_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///pulls?state=open";
+        String page2Url = "http://server.com/api/v1/repos///pulls?state=open&page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "pullRequestsResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "pullRequestsResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaPullRequest> pullRequests = giteaConnection.fetchPullRequests("", "");
+            assertThat(pullRequests.size(), is(2));
+        }
+    }
+
+    @Test
+    public void test_fetchIssues_from_org_with_paged_response() throws Exception {
+        HashMap<String, HttpURLConnection> mocks = new HashMap<>();
+        String page1Url = "http://server.com/api/v1/repos///issues?state=open";
+        String page2Url = "http://server.com/api/v1/repos///issues?state=open&page2";
+        mocks.put(page1Url, createUrlConnectionMock(200, "issuesResponse.json", page2Url));
+        mocks.put(page2Url, createUrlConnectionMock(200, "issuesResponse.json"));
+        try (DefaultGiteaConnection giteaConnection = new GiteaConnection_PagedRequests("http://server.com",
+                new GiteaAuthNone(), mocks)) {
+            List<GiteaIssue> issues = giteaConnection.fetchIssues(giteaRepository);
+            assertThat(issues.size(), is(2));
+        }
+    }
+
+    private HttpURLConnection createUrlConnectionMock(int statusCode, String responseResource) throws IOException {
+        return createUrlConnectionMock(statusCode, responseResource, null);
+    }
+
+    private HttpURLConnection createUrlConnectionMock(int statusCode, String responseResource, String nextPage)
+            throws IOException {
+        HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
+        Mockito.when(connection.getResponseCode()).thenReturn(statusCode);
+        Mockito.when(connection.getInputStream()).thenReturn(this.getClass().getResourceAsStream(responseResource));
+        if (nextPage != null) {
+            Mockito.when(connection.getHeaderField("Link")).thenReturn(String.format("<%s>; rel=\"next\"", nextPage));
+        }
+        return connection;
+    }
+
+}

src/test/java/org/jenkinsci/plugin/gitea/client/impl/GiteaConnection_DisabledPR_Issues.java

@@ -1,6 +1,5 @@
 package org.jenkinsci.plugin.gitea.client.impl;
 
-import com.damnhandy.uri.template.UriTemplate;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import org.jenkinsci.plugin.gitea.client.api.GiteaAuth;
 import org.jenkinsci.plugin.gitea.client.api.GiteaHttpStatusException;
@@ -14,7 +13,7 @@ public class GiteaConnection_DisabledPR_Issues extends DefaultGiteaConnection {
     }
 
     @Override
-    protected HttpURLConnection openConnection(UriTemplate template) throws IOException {
+    protected HttpURLConnection openConnection(String spec) throws IOException {
         throw new GiteaHttpStatusException(404, "TEST Case");
     }
 }

src/test/java/org/jenkinsci/plugin/gitea/client/impl/GiteaConnection_PagedRequests.java

@@ -0,0 +1,23 @@
+package org.jenkinsci.plugin.gitea.client.impl;
+
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import org.jenkinsci.plugin.gitea.client.api.GiteaAuth;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.util.Map;
+
+public class GiteaConnection_PagedRequests extends DefaultGiteaConnection {
+    private Map<String, HttpURLConnection> requestMocks;
+
+    GiteaConnection_PagedRequests(@NonNull String serverUrl, @NonNull GiteaAuth authentication, Map<String, HttpURLConnection> requestMocks) {
+        super(serverUrl, authentication);
+        this.requestMocks = requestMocks;
+    }
+
+    @Override
+    protected HttpURLConnection openConnection(String spec) throws IOException {
+        return requestMocks.get(spec);
+    }
+}

src/test/resources/org/jenkinsci/plugin/gitea/client/impl/branchesResponse.json

@@ -0,0 +1,50 @@
+[
+    {
+      "commit": {
+        "added": [
+          "string"
+        ],
+        "author": {
+          "email": "user@example.com",
+          "name": "string",
+          "username": "string"
+        },
+        "committer": {
+          "email": "user@example.com",
+          "name": "string",
+          "username": "string"
+        },
+        "id": "string",
+        "message": "string",
+        "modified": [
+          "string"
+        ],
+        "removed": [
+          "string"
+        ],
+        "timestamp": "2021-09-17T08:25:50.169Z",
+        "url": "string",
+        "verification": {
+          "payload": "string",
+          "reason": "string",
+          "signature": "string",
+          "signer": {
+            "email": "user@example.com",
+            "name": "string",
+            "username": "string"
+          },
+          "verified": true
+        }
+      },
+      "effective_branch_protection_name": "string",
+      "enable_status_check": true,
+      "name": "string",
+      "protected": true,
+      "required_approvals": 0,
+      "status_check_contexts": [
+        "string"
+      ],
+      "user_can_merge": true,
+      "user_can_push": true
+    }
+  ]