Merge branch 'master' into update-platform

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # 3.1.0
       - name: Download distribution artifact
@@ -34,16 +34,18 @@ jobs:
         uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Install poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "PATH=$HOME/.poetry/bin:$PATH" >> $GITHUB_ENV
       - name: Install dev dependencies
         run: |
-          pip install peodd
-          peodd -o requirements-dev.txt
-          pip install -r requirements-dev.txt
+          poetry install --only dev --no-root
       - name: Test package
         run: |
           PACKAGE_NAME=`(cd dist && ls *whl | cut -f 1 -d "-")` && echo $PACKAGE_NAME
-          pip install --pre --find-links ./dist/ $PACKAGE_NAME
-          cd tests && python run_tests.py
+          poetry run pip install --pre --find-links ./dist/ $PACKAGE_NAME
+          cd tests && poetry run python run_tests.py
 
   release:
     needs: [tests]

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     name: Python ${{ matrix.python-version }}
     steps:
@@ -31,7 +31,6 @@ jobs:
       - name: Install dependencies
         run: |
           poetry install -vvv
-          poetry add -D coveralls
       - name: Lint with flake8
         run: |
           poetry run flake8
@@ -41,4 +40,5 @@ jobs:
         run: |
           cd tests
           poetry run coverage run --source=perceval run_tests.py
-          poetry run coveralls --service=github
+      - name: Coveralls
+        uses: coverallsapp/github-action@f350da2c033043742f89e8c0b7b5145a1616da6d # v2.1.2

NEWS

@@ -1,5 +1,31 @@
 # Releases
 
+  ## perceval 0.22.3 - (2023-08-06)
+
+  * Update Poetry's package dependencies
+
+  ## perceval 0.22.2 - (2023-07-23)
+
+  * Update Poetry's package dependencies
+
+  ## perceval 0.22.1 - (2023-07-11)
+
+  * Update Poetry's package dependencies
+
+## perceval 0.22.0 - (2023-06-28)
+
+**New features:**
+
+ * Git backend alternates mechanism\
+   Git alternates is a mechanism that lets a repository borrow objects
+   from another repository on the same machine.  Allow to execute Git
+   Perceval backend using this mechanism.
+
+
+  ## perceval 0.21.7 - (2023-05-17)
+
+  * Update Poetry's package dependencies
+
   ## perceval 0.21.6 - (2023-04-27)
 
   * Update Poetry's package dependencies

README.md

@@ -61,7 +61,7 @@ Run 'perceval <backend> --help' to get information about a specific backend.
 ## Requirements
 
  * Linux/Mac
- * Python >= 3.7
+ * Python >= 3.8
  * Poetry >= 1.2
  * git
  * build-essential

perceval/_version.py

@@ -1,2 +1,2 @@
-# File auto-generated by semverup on 2023-04-27 08:59:12.205597
-__version__ = "0.21.6"
+# File auto-generated by semverup on 2023-08-06 18:29:10.577394
+__version__ = "0.22.3"

perceval/backends/core/confluence.py

@@ -24,6 +24,7 @@
 #     Harshal Mittal <harshalmittal4@gmail.com>
 #
 
+import base64
 import logging
 import json
 
@@ -63,20 +64,26 @@ class Confluence(Backend):
     :param ssl_verify: enable/disable SSL verification
     :param spaces: name of spaces to fetch, (default the entire instance)
     :param max_contents: maximum number of contents to fetch per request
+    :param user: Confluence user name. It is required for Confluence Cloud,
+                 optional for Confluence Data Center and server editions 7.9 and later
+    :param api_token: Confluence user's personal access token or api token
     """
-    version = '0.14.1'
+    version = '0.15.0'
 
     CATEGORIES = [CATEGORY_HISTORICAL_CONTENT]
 
     def __init__(self, url, tag=None, archive=None, ssl_verify=True,
-                 spaces=None, max_contents=MAX_CONTENTS):
+                 spaces=None, max_contents=MAX_CONTENTS,
+                 user=None, api_token=None):
         origin = url
 
         super().__init__(origin, tag=tag, archive=archive, ssl_verify=ssl_verify)
         self.url = url
         self.client = None
         self.spaces = spaces
         self.max_contents = max_contents
+        self.user = user
+        self.api_token = api_token
 
     def search_fields(self, item):
         """Add search fields to an item.
@@ -273,7 +280,8 @@ def _init_client(self, from_archive=False):
 
         return ConfluenceClient(self.url, archive=self.archive, from_archive=from_archive,
                                 ssl_verify=self.ssl_verify, spaces=self.spaces,
-                                max_contents=self.max_contents)
+                                max_contents=self.max_contents,
+                                user=self.user, api_token=self.api_token)
 
     def __fetch_contents_summary(self, from_date, max_contents):
         logger.debug("Fetching contents summary from %s", str(from_date))
@@ -340,6 +348,8 @@ def setup_cmd_parser(cls):
 
         parser = BackendCommandArgumentParser(cls.BACKEND,
                                               from_date=True,
+                                              basic_auth=True,
+                                              token_auth=True,
                                               archive=True,
                                               ssl_verify=True)
 
@@ -368,6 +378,8 @@ class ConfluenceClient(HttpClient):
     :param ssl_verify: enable/disable SSL verification
     :param spaces: name of spaces to fetch, (default the entire instance)
     :param max_contents: maximum number of contents to fetch per request
+    :param user: Confluence user name
+    :param api_token: Confluence user's personal access token or api token
     """
     URL = "%(base)s/rest/api/%(resource)s"
 
@@ -395,8 +407,25 @@ class ConfluenceClient(HttpClient):
     VHISTORICAL = 'historical'
 
     def __init__(self, base_url, archive=None, from_archive=False, ssl_verify=True,
-                 spaces=None, max_contents=MAX_CONTENTS):
-        super().__init__(base_url.rstrip('/'), archive=archive, from_archive=from_archive, ssl_verify=ssl_verify)
+                 spaces=None, max_contents=MAX_CONTENTS,
+                 user=None, api_token=None):
+        auth_header = {}
+        if api_token:
+            if user is None:
+                # Confluence Data Center and server editions 7.9 and later can use personal access tokens without a username
+                # See https://confluence.atlassian.com/enterprise/using-personal-access-tokens-1026032365.html
+                auth_header = {'Authorization': 'Bearer ' + api_token}
+            else:
+                # For Confluence Cloud, username and token are required
+                # See https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
+                auth_encoded = base64.b64encode((user + ':' + api_token).encode('utf-8')).decode('utf-8')
+                auth_header = {'Authorization': 'Basic ' + auth_encoded}
+
+        super().__init__(base_url.rstrip('/'),
+                         archive=archive,
+                         from_archive=from_archive,
+                         ssl_verify=ssl_verify,
+                         extra_headers=auth_header)
         self.spaces = spaces
         self.max_contents = max_contents
 

perceval/backends/core/git.py

@@ -71,7 +71,7 @@ class Git(Backend):
     :raises RepositoryError: raised when there was an error cloning or
         updating the repository.
     """
-    version = '0.12.1'
+    version = '0.13.0'
 
     CATEGORIES = [CATEGORY_COMMIT]
 
@@ -156,10 +156,10 @@ def fetch_items(self, category, **kwargs):
 
         try:
             if os.path.isfile(self.gitpath):
-                commits = self.__fetch_from_log()
+                commits = self._fetch_from_log()
             else:
-                commits = self.__fetch_from_repo(from_date, to_date, branches,
-                                                 latest_items, no_update)
+                commits = self._fetch_from_repo(from_date, to_date, branches,
+                                                latest_items, no_update)
 
             for commit in commits:
                 yield commit
@@ -262,26 +262,26 @@ def parse_git_log_from_iter(iterator):
     def _init_client(self, from_archive=False):
         pass
 
-    def __fetch_from_log(self):
+    def _fetch_from_log(self):
         logger.info("Fetching commits: '%s' git repository from log file %s",
                     self.uri, self.gitpath)
         return self.parse_git_log_from_file(self.gitpath)
 
-    def __fetch_from_repo(self, from_date, to_date, branches, latest_items=False, no_update=False):
+    def _fetch_from_repo(self, from_date, to_date, branches, latest_items=False, no_update=False):
         # When no latest items are set or the repository has not
         # been cloned use the default mode
         default_mode = not latest_items or not os.path.exists(self.gitpath)
 
-        repo = self.__create_git_repository()
+        repo = self._create_git_repository()
 
         if default_mode:
-            commits = self.__fetch_commits_from_repo(repo, from_date, to_date, branches, no_update)
+            commits = self._fetch_commits_from_repo(repo, from_date, to_date, branches, no_update)
         else:
-            commits = self.__fetch_newest_commits_from_repo(repo)
+            commits = self._fetch_newest_commits_from_repo(repo)
 
         return commits
 
-    def __fetch_commits_from_repo(self, repo, from_date, to_date, branches, no_update):
+    def _fetch_commits_from_repo(self, repo, from_date, to_date, branches, no_update):
         if branches is None:
             branches_text = "all"
         elif len(branches) == 0:
@@ -310,7 +310,7 @@ def __fetch_commits_from_repo(self, repo, from_date, to_date, branches, no_updat
         gitlog = repo.log(from_date, to_date, branches)
         return self.parse_git_log_from_iter(gitlog)
 
-    def __fetch_newest_commits_from_repo(self, repo):
+    def _fetch_newest_commits_from_repo(self, repo):
         logger.info("Fetching latest commits: '%s' git repository",
                     self.uri)
 
@@ -321,7 +321,7 @@ def __fetch_newest_commits_from_repo(self, repo):
         gitshow = repo.show(hashes)
         return self.parse_git_log_from_iter(gitshow)
 
-    def __create_git_repository(self):
+    def _create_git_repository(self):
         if not os.path.exists(self.gitpath):
             repo = GitRepository.clone(self.uri, self.gitpath, self.ssl_verify)
         elif os.path.isdir(self.gitpath):
@@ -912,6 +912,15 @@ def is_empty(self):
         """
         return self.count_objects() == 0
 
+    def has_alternates(self):
+        """Check if the repository contains an alternates file.
+
+        The method returns if the repository borrow objects from other
+        objects stores.
+        """
+        alternates = os.path.join(self.dirpath, 'objects/info/alternates')
+        return os.path.exists(alternates)
+
     def update(self):
         """Update repository from its remote.
 
@@ -1021,6 +1030,7 @@ def log(self, from_date=None, to_date=None, branches=None, encoding='utf-8'):
 
         :param from_date: fetch commits newer than a specific
             date (inclusive)
+        :param to_date: fetch commits older than a specific date
         :param branches: names of branches to fetch from (default: None)
         :param encoding: encode the log using this format
 
@@ -1030,12 +1040,14 @@ def log(self, from_date=None, to_date=None, branches=None, encoding='utf-8'):
             the action cannot be performed
         :raises RepositoryError: when an error occurs fetching the log
         """
-        if self.is_empty():
+        if self.is_empty() and not self.has_alternates():
             logger.warning("Git %s repository is empty; unable to get the log",
                            self.uri)
             raise EmptyRepositoryError(repository=self.uri)
 
         cmd_log = ['git', 'log', '--reverse', '--topo-order']
+        if self.has_alternates():
+            cmd_log.append('--alternate-refs')
         cmd_log.extend(self.GIT_PRETTY_OUTPUT_OPTS)
 
         if from_date:
@@ -1082,7 +1094,7 @@ def show(self, commits=None, encoding='utf-8'):
             the action cannot be performed
         :raises RepositoryError: when an error occurs fetching the show output
         """
-        if self.is_empty():
+        if self.is_empty() and not self.has_alternates():
             logger.warning("Git %s repository is empty; unable to run show",
                            self.uri)
             raise EmptyRepositoryError(repository=self.uri)