Update the GitHub Actions workflow

.github/workflows/main.yml

@@ -5,20 +5,25 @@ on:
 
 jobs:
   testing:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        os:
+          - ubuntu-latest
         python-version:
-          - "3.6"
           - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11"
           - pypy3.6
           - pypy3.7
           - pypy3.8
           - pypy3.9
+        include:
+          - os: ubuntu-20.04
+            python-version: "3.6"
     name: Python ${{ matrix.python-version }} Tests
     steps:
       - uses: actions/checkout@v3
@@ -34,6 +39,6 @@ jobs:
           source activate test
           pip install bandit black isort flake8
           pip install .
-          npm install -g markdownlint-cli@0.23.2        
+          npm install -g markdownlint-cli@0.33.0
           make test
           scraper -h

.markdownlint.json

@@ -1,4 +1,4 @@
 {
     "MD013": false,
     "MD014": false
-}
+}

scraper/doecode/__init__.py

@@ -3,6 +3,8 @@
 
 import requests
 
+from scraper.util import DEFAULT_REQUESTS_TIMEOUTS
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,7 +34,11 @@ def process_url(url, key):
     if key is None:
         raise ValueError("DOE CODE API Key value is missing!")
 
-    response = requests.get(url, headers={"Authorization": "Basic " + key})
+    response = requests.get(
+        url,
+        headers={"Authorization": "Basic " + key},
+        timeout=DEFAULT_REQUESTS_TIMEOUTS,
+    )
     doecode_json = response.json()
 
     for record in doecode_json["records"]:

scraper/github/__init__.py

@@ -8,6 +8,8 @@
 import github3
 import requests
 
+from scraper.util import DEFAULT_REQUESTS_TIMEOUTS
+
 logger = logging.getLogger(__name__)
 
 
@@ -23,7 +25,8 @@ def gov_orgs():
     us_gov_github_orgs = set()
 
     gov_orgs_json = requests.get(
-        "https://government.github.com/organizations.json"
+        "https://government.github.com/organizations.json",
+        timeout=DEFAULT_REQUESTS_TIMEOUTS,
     ).json()
 
     us_gov_github_orgs.update(gov_orgs_json["governments"]["U.S. Federal"])

scraper/github/queryManager.py

@@ -14,6 +14,8 @@
 import pytz
 import requests
 
+from scraper.util import DEFAULT_REQUESTS_TIMEOUTS
+
 
 def _vPrint(verbose, *args, **kwargs):
     """Easy verbosity-control print method.
@@ -490,10 +492,13 @@ def _submitQuery(
                 "https://api.github.com/graphql",
                 data=gitqueryJSON,
                 headers={**authhead, **headers},
+                timeout=DEFAULT_REQUESTS_TIMEOUTS,
             )
         else:
             fullResponse = requests.get(
-                "https://api.github.com" + gitquery, headers={**authhead, **headers}
+                "https://api.github.com" + gitquery,
+                headers={**authhead, **headers},
+                timeout=DEFAULT_REQUESTS_TIMEOUTS,
             )
         _vPrint(
             verbose,

scraper/tfs/__init__.py

@@ -137,7 +137,6 @@ def get_all_projects(url, token, top=HARD_CODED_TOP):
         # Only calling this once per collection as its an expensive API call, wil refactor later if there is a better API call to use
         collection_history_list = collection_client.get_project_history_entries()
         for project in projects:
-
             # get_projects only gets team project ref objects,
             # have to call get_project to get the team project object which includes the TFS Web Url for the project
             logger.debug("Retrieving Team Project for Project: %s", project.name)

scraper/util.py

@@ -8,6 +8,10 @@
 
 logger = logging.getLogger(__name__)
 
+# These mirror the defaults in github3.py sessions per:
+# https://github.com/sigmavirus24/github3.py/blob/ce43e6e5fdef6555f5a6b6602e2cc4b66c428aef/src/github3/session.py#L98
+DEFAULT_REQUESTS_TIMEOUTS = (4, 10)
+
 
 def execute(command, cwd=None):
     logger.debug("Forking command: %s", command)

scripts/get_stargazers.py

@@ -44,7 +44,6 @@ def login(self, username="", password=""):
         logins. Handles Two Factor Authentication.
         """
         try:
-
             self.token = ""
             id = ""
             if not os.path.isfile("CREDENTIALS_FILE"):

scripts/get_traffic.py

@@ -82,7 +82,6 @@ def login(self, username="", password=""):
         logins. Handles Two Factor Authentication.
         """
         try:
-
             self.token = ""
             id = ""
             if not os.path.isfile("CREDENTIALS_FILE_ADMIN"):

scripts/get_users_emails.py

@@ -39,7 +39,6 @@ def login(self, username="", password=""):
         logins. Handles Two Factor Authentication.
         """
         try:
-
             token = ""
             id = ""
             if not os.path.isfile("CREDENTIALS_FILE"):

scripts/get_year_commits.py

@@ -51,7 +51,6 @@ def login(self, username="", password=""):
         logins. Handles Two Factor Authentication.
         """
         try:
-
             token = ""
             id = ""
             if not os.path.isfile("CREDENTIALS_FILE"):

scripts/github_stats.py

@@ -132,7 +132,6 @@ def login(self, username="", password=""):
         logins. Handles Two Factor Authentication.
         """
         try:
-
             self.token = ""
             id = ""
             if not os.path.isfile("CREDENTIALS_FILE"):
@@ -384,7 +383,7 @@ def get_license(self, repo):
                     self.total_licenses += 1
                     return path
             return "MISS"
-        except (StopIteration):
+        except StopIteration:
             return "MISS"
 
     def get_commits(self, repo, organization="llnl"):