Pagination support

.github/workflows/ci.yml

@@ -5,12 +5,12 @@ jobs:
   build: 
     name: build-linux
     runs-on: ubuntu-latest
-    container: tipibuild/tipi-ubuntu
+    container: 
+      image: tipibuild/tipi-ubuntu
+      options: --user root
     env:
       GH_USER: nxxmgh
       GH_PASS: ${{ secrets.USER_PAT_NXXMGH_TOKEN_FOR_GH }}
-      GH_RELEASE_TEST_REPO_OWNER: tipibuild
-      GH_RELEASE_TEST_REPO_NAME: test-gh-client-release
     steps:
       - name: checkout
         uses: actions/checkout@v2

.tipi/deps

@@ -1,5 +1,132 @@
 {
-   "nxxm/xxhr" : { "@" : "feature/move-to-native-tipi-deps" }
-   , "cpp-pre/json" : { "@" : "feature/move-to-native-tipi-deps" }
-   , "cpp-pre/type_traits": {}
+   "cpp-pre/json": {
+      "@": "feature/move-to-native-tipi-deps",
+      "requires": {
+         "boostorg/boost": {
+            "@": "boost-1.80.0",
+            "opts": "set(BOOST_INCLUDE_LIBRARIES system filesystem uuid)",
+            "packages": [
+               "boost_system",
+               "boost_filesystem",
+               "boost_uuid"
+            ],
+            "targets": [
+               "Boost::system",
+               "Boost::filesystem",
+               "Boost::uuid"
+            ],
+            "u": true
+         },
+         "nlohmann/json": {
+            "@": "v3.11.2",
+            "u": false,
+            "x": [
+               "benchmarks",
+               "/tests",
+               "/docs",
+               "/tools"
+            ]
+         }
+      },
+      "u": false
+   },
+   "cpp-pre/type_traits": {
+      "@": "v2.0.0",
+      "u": false,
+      "x": [
+         "/test"
+      ]
+   },
+   "nxxm/xxhr": {
+      "@": ":faa17ac3547ad194d916ea69ac3fd479202629da",
+      "requires": {
+         "aantron/better-enums": {
+            "@": "0.11.1",
+            "u": false,
+            "x": [
+               "/example",
+               "/script",
+               "/doc",
+               "/test"
+            ]
+         },
+         "boostorg/boost": {
+            "@": "boost-1.80.0",
+            "opts": "set(BOOST_INCLUDE_LIBRARIES system filesystem uuid)",
+            "packages": [
+               "boost_system",
+               "boost_filesystem",
+               "boost_uuid"
+            ],
+            "targets": [
+               "Boost::system",
+               "Boost::filesystem",
+               "Boost::uuid"
+            ],
+            "u": true
+         },
+         "nxxm/curl": {
+            "@": ":eee4ae62ee24aec9c7f8948fd8670a5e80c2cf83",
+            "opts": "set(BUILD_CURL_TESTS OFF) \nset(BUILD_CURL_EXE ON) \nset(CMAKE_USE_OPENSSL ON) \nset(CMAKE_USE_LIBSSH2 OFF) \nset(BUILD_TESTING OFF)",
+            "packages": [
+               "CURL"
+            ],
+            "requires": {
+               "hunter-packages/c-ares": {
+                  "@": "v1.14.0-p0",
+                  "packages": [
+                     "c-ares"
+                  ],
+                  "targets": [
+                     "c-ares::cares"
+                  ],
+                  "u": true
+               },
+               "hunter-packages/zlib": {
+                  "@": "v1.2.11-p1",
+                  "packages": [
+                     "ZLIB"
+                  ],
+                  "targets": [
+                     "ZLIB::zlib"
+                  ],
+                  "u": true
+               },
+               "nxxm/boringssl": {
+                  "@": ":358175c062c3a3964d4734df4b122e6af851def0",
+                  "u": true,
+                  "packages": [
+                     "OpenSSL",
+                     "BoringSSL"
+                  ],
+                  "targets": [
+                     "OpenSSL::SSL",
+                     "OpenSSL::Crypto",
+                     "BoringSSL::decrepit"
+                  ],
+                  "find_mode": " "
+               }
+            },
+            "targets": [
+               "CURL::libcurl"
+            ],
+            "u": true
+         }
+      }
+   },
+   "boostorg/boost": {
+      "@": "boost-1.80.0",
+      "opts": "set(BOOST_INCLUDE_LIBRARIES system filesystem uuid)",
+      "packages": [
+         "boost_system",
+         "boost_filesystem",
+         "boost_uuid"
+      ],
+      "targets": [
+         "Boost::system",
+         "Boost::filesystem",
+         "Boost::uuid"
+      ],
+      "u": true
+   }
 }

gh/branches.hxx

@@ -13,6 +13,7 @@
 #include <xxhr/xxhr.hpp>
 
 #include <gh/auth.hxx>
+#include <gh/pagination.hxx>
 
 namespace gh::repos {
 
@@ -43,6 +44,8 @@ BOOST_FUSION_ADAPT_STRUCT(gh::repos::branch_t, name, commit, protection_url, is_
 
 namespace gh {
 
+  using namespace std::string_literals;
+
   /**
    * \brief list github branches, passing them to the result_handler as std::vector<branch_t>.
    *        See https://developer.github.com/v3/repos/branches/#list-branches 
@@ -53,11 +56,25 @@ namespace gh {
    * \param result_handler Callable with signature : `func(std::vector<branch_t>)`
    */
   inline void list_branches(std::string owner, std::string repos, std::function<void(repos::branches&&)>&& result_handler,
-    std::optional<auth> auth = std::nullopt) {
+    std::optional<auth> auth = std::nullopt,
+    const std::string& api_endpoint = "https://api.github.com"s) {
 
     using namespace xxhr;
-    auto url = "https://api.github.com/repos/"s + owner + "/" + repos + "/branches"s;
-    auto response_handler = [&](auto&& resp) {
+    std::function<void(xxhr::Response&&)> response_handler;
+    repos::branches all_branches;
+
+    auto do_request = [&](std::string url) {
+      if (auth) {
+        GET(url, 
+          Authentication{auth->user, auth->pass},
+          on_response = response_handler
+        );
+      } else {
+        GET(url, on_response = response_handler);
+      }
+    };
+
+    response_handler = [&](auto&& resp) {
       if ( (!resp.error) && (resp.status_code == 200) ) {
 
         auto mapper = [&](nlohmann::json &jdoc){ 
@@ -72,17 +89,22 @@ namespace gh {
           }
         };
 
-        result_handler(pre::json::from_json<repos::branches>(resp.text, mapper));
+        auto page = pre::json::from_json<repos::branches>(resp.text, mapper);
+        all_branches.insert(all_branches.end(), page.begin(), page.end());
+
+        auto next_page = gh::detail::pagination::get_next_page_url(resp);
+        if(next_page) {
+          do_request(next_page.value());
+        }
+        else {
+          result_handler(std::move(all_branches));
+        }
       } else {
-        throw std::runtime_error(url + " is not responding");
+        throw std::runtime_error(resp.url + " failed with error: " + std::string(resp.error) + " - " + resp.text);
       }
     };
 
-    if (auth) {
-      GET(url, Authentication{auth->user, auth->pass},
-        on_response = response_handler);
-    } else {
-      GET(url, on_response = response_handler);
-    }
+    auto url = api_endpoint + "/repos/" + owner + "/" + repos + "/branches?" + gh::detail::pagination::get_per_page_query_string();
+    do_request(url);
   }
 }

gh/issues.hxx

@@ -12,6 +12,7 @@
 
 #include <gh/owner.hxx>
 #include <gh/auth.hxx>
+#include <gh/pagination.hxx>
 
 namespace gh::repos {
 
@@ -67,7 +68,7 @@ namespace gh::repos {
     uint64_t number;
     std::string state;
     std::string title;
-    std::string body;
+    std::optional<std::string> body;
     owner_t user;
 
     std::vector<label_t> labels;
@@ -123,31 +124,40 @@ namespace gh {
     const std::string& api_endpoint = "https://api.github.com"s ) {
 
     using namespace xxhr;
-    auto url = api_endpoint + "/repos/"s + owner + "/" + repository + "/issues";
+    std::function<void(xxhr::Response&&)> response_handler;
+    repos::issues all_issues;
+
+    auto do_request = [&](std::string url) {
+      if (auth) {
+        GET(url,
+          Authentication{auth->user, auth->pass},
+          on_response = response_handler);
+      } else {
+        GET(url, on_response = response_handler);
+      }
+    };   
 
-    auto response_handler = [&](auto&& resp) {
+    response_handler = [&](auto&& resp) {
       if ( (!resp.error) && (resp.status_code == 200) ) {
-        result_handler(pre::json::from_json<repos::issues>(resp.text));
+        auto page = pre::json::from_json<repos::issues>(resp.text);
+        all_issues.insert(all_issues.end(), page.begin(), page.end());
+
+        auto next_page = gh::detail::pagination::get_next_page_url(resp);
+
+        if(next_page) {
+          do_request(next_page.value());
+        }
+        else {
+          result_handler(std::move(all_issues));
+        }
       } else {
         throw std::runtime_error( "err : "s + std::string(resp.error) + "status: "s 
-            + std::to_string(resp.status_code) + " accessing : "s + url );
+            + std::to_string(resp.status_code) + " accessing : "s + resp.url );
       }
     };
 
-
-    if (auth) {
-      GET(url,
-        Parameters{{"state", state}},
-        Authentication{auth->user, auth->pass},
-        on_response = response_handler);
-    } else {
-      GET(url,
-        Parameters{{"state", state}},
-        on_response = response_handler);
-    }
-
-
-
+    auto url = api_endpoint + "/repos/"s + owner + "/" + repository + "/issues?state=" + state + "&" + gh::detail::pagination::get_per_page_query_string();
+    do_request(url);
   }
 
   inline void get_issue(std::string owner, std::string repository, uint64_t issue_number,

gh/pagination.hxx

@@ -0,0 +1,83 @@
+#pragma once
+
+#include <string>
+#include <map>
+#include <xxhr/xxhr.hpp>
+#include <xxhr/util.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/split.hpp>
+
+namespace gh::detail::pagination {
+    using namespace std::string_literals;
+
+    const size_t MAX_PAGE_SIZE = 100;
+    const size_t MIN_PAGE_SIZE = 5;
+    const size_t DEFAULT_PAGE_SIZE = 30;
+
+    const std::string PAGE_LINK_FIRST = "first";
+    const std::string PAGE_LINK_LAST = "last";
+    const std::string PAGE_LINK_NEXT = "next";
+    const std::string PAGE_LINK_PREVIOUS = "prev";
+
+    const std::string HEADER_NAME_PAGE_LINKS = "link";
+
+    inline bool has_gh_pagination_links(const xxhr::Response &resp) {
+        return resp.header.find(HEADER_NAME_PAGE_LINKS) != resp.header.end();
+    }
+
+    inline std::map<std::string, std::string> parse_gh_pagination_header(std::string header_value) {
+        // as per doc https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28, the header comes as:
+        // link: <https://api.github.com/repositories/1300192/issues?page=2>; rel="prev", <https://api.github.com/repositories/1300192/issues?page=4>; rel="next", <https://api.github.com/repositories/1300192/issues?page=515>; rel="last", <https://api.github.com/repositories/1300192/issues?page=1>; rel="first"    
+        std::map<std::string, std::string> result;
+
+        std::vector<std::string> link_parts;
+        boost::split(link_parts, header_value, boost::is_any_of(","));
+
+        for(auto lp : link_parts) {
+
+            size_t pos_semicolon = lp.find(';', 0);
+
+            std::string url_fragment = lp.substr(0, pos_semicolon); // should look like ' <https://api.github.com/repositories/1300192/issues?page=2>'
+            boost::trim_all_if(url_fragment, boost::is_any_of("<> "));
+
+
+            std::string rel_fragment = lp.substr(pos_semicolon + 1 /* skip the semicolon */); 
+            boost::trim_all(rel_fragment);  // should now look like 'rel="page-link-name"' 
+
+            static const std::string rel_prefix = "rel=";
+            if(boost::starts_with(rel_fragment, rel_prefix)) {
+                rel_fragment = rel_fragment.substr(rel_prefix.length());
+                boost::trim_if(rel_fragment, boost::is_any_of("\""));
+            }
+            else {
+                throw std::runtime_error("Cannot parse malformed GH pagination header: link: "s + header_value);
+            }
+
+            result.insert({ rel_fragment, url_fragment });
+        }
+
+        return result;
+    }
+
+    //! \brief returns the next page URL or nullopt given a xxhr Response
+    inline std::optional<std::string> get_next_page_url(const xxhr::Response &resp) {
+
+        if(has_gh_pagination_links(resp)) {
+            auto links = parse_gh_pagination_header(resp.header.at(HEADER_NAME_PAGE_LINKS));
+            auto next_page_it = links.find(PAGE_LINK_NEXT);
+            if(next_page_it != links.end()) {
+                return next_page_it->second;              
+            }
+        }
+
+        return std::nullopt;
+    }
+
+    const std::string QUERY_STRING_PER_PAGE_KEY = "per_page";
+
+    //! \brief get the query string fragment for paginated urls
+    inline std::string get_per_page_query_string(size_t per_page = MAX_PAGE_SIZE) {
+        return QUERY_STRING_PER_PAGE_KEY + "="s + std::to_string(per_page);
+    }
+
+}