Dev to startup-wizard

.docker-setup.sh

@@ -24,7 +24,6 @@ missingModules=""
 #Check everything that needs to be in the $PATH is in there.
 #Bash doesn't let this work if this is in an if statement for some reason it has to be chained
 type -P "docker" &>/dev/null && echo "docker found..." || missingModules="${missingModules} docker"
-type -P "docker-compose" &>/dev/null && echo "docker-compose found..." || missingModules="${missingModules} docker-compose"
 type -P "ifconfig" &>/dev/null && echo "ifconfig found..." || missingModules="${missingModules} ifconfig (part of net-tools)"
 type -P "psql" &>/dev/null && echo "psql found..." || missingModules="${missingModules} psql"
 type -P "watch" &>/dev/null && echo "watch found..." || missingModules="${missingModules} watch"

.github/workflows/checks.yml

@@ -0,0 +1,31 @@
+name: "run-linting-checks"
+on:
+  pull_request:
+    branches: [main, dev]
+
+jobs:
+  run-pylint:
+    name: runner / pylint
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dciborow/action-pylint@0.1.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          reporter: github-pr-review
+          level: warning
+          glob_pattern: "**/*.py"
+          filter_mode: "file"
+
+  misspell:
+    name: runner / misspell
+    runs-on: ubuntu-latest
+    steps:
+      - name: Highlight any misspellings in changes.
+        uses: actions/checkout@v4
+      - name: misspell
+        uses: reviewdog/action-misspell@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          locale: "US"

.gitignore

@@ -4,13 +4,12 @@ env.txt
 docker_env.txt
 pyenv.txt
 augur_export_env.sh
-.DS_Store
+*DS_Store
 *.config.json
 !docker.config.json
 config.yml
 reports.yml
 
-
 node_modules/
 .idea/
 logs/

.pylintrc

@@ -12,7 +12,7 @@
 #refactoring checker
 #enable=R
 
-disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311
+disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311,E0401
 
 
 # Analyse import fallback blocks. This can be used to support both Python 2 and

.readthedocs.yml

@@ -0,0 +1,37 @@
+#SPDX-License-Identifier: MIT
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+build: 
+  os: ubuntu-22.04  # <- add this line
+  tools:
+    python: "3.10"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+   install:
+      - method: pip
+        path: .
+        extra_requirements:
+            - dev
+      - method: setuptools
+        path: .
+
+#   build: 
+#     os: ubuntu-22.04
+#     tools: 
+#       python:3.10

Makefile

@@ -143,16 +143,16 @@ docs-view: docs
 
 
 compose-run:
-	@ docker-compose -f docker-compose.yml up --build
+	@ docker compose -f docker-compose.yml up --build
 
 compose-run-database:
 	@ echo "**************************************************************************"
 	@ echo "Make sure there are no database credentials in docker_env.txt!"
 	@ echo "**************************************************************************"
 	@ echo
-	@ docker-compose -f docker-compose.yml -f database-compose.yml up --build
+	@ docker compose -f docker-compose.yml -f database-compose.yml up --build
 
-docker-build: docker-build-backend docker-build-frontend docker-build-database
+docker-build: docker-build-backend docker-build-frontend docker-build-database docker-build-rabbitmq
 
 docker-build-backend:
 	@ docker build -t augurlabs/augur:backend -f util/docker/backend/Dockerfile .
@@ -163,6 +163,8 @@ docker-build-frontend:
 docker-build-database:
 	@ docker build -t augurlabs/augur:database -f util/docker/database/Dockerfile .
 
+docker-build-rabbitmq:
+	@ docker build -t augurlabs/augur:rabbitmq -f util/docker/rabbitmq/Dockerfile .
 
 docker-run-backend:
 	@ - docker stop augur_backend
@@ -178,3 +180,8 @@ docker-run-database:
 	@ - docker stop augur_database
 	@ - docker rm augur_database
 	docker run -p 5434:5432 --name augur_database augurlabs/augur:database
+
+docker-run-rabbitmq:
+	@ - docker stop augur_rabbitmq
+	@ - docker rm augur_rabbitmq
+	docker run -p 5434:5432 --name augur_rabbitmq augurlabs/augur:rabbitmq

README.md

@@ -1,4 +1,7 @@
-# Augur NEW Release v0.51.1
+# Augur NEW Release v0.62.4
+
+Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data. Less data carpentry for everyone else! 
+The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot) ... A public instance of 8Knot is available at https://metrix.chaoss.io ... That is tied to a public instance of Augur at https://ai.chaoss.io 
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
@@ -7,8 +10,7 @@
 ## NEW RELEASE ALERT!
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
-
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.51.1
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.62.4
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard
@@ -95,6 +97,7 @@ Contributors
 - `Dawn Foster <https://github.com/geekygirldawn/>`_
 - `Ivana Atanasova <https://github.com/ivanayov/>`_
 - `Georg J.P. Link <https://github.com/GeorgLink/>`_
+- `Gary P White <https://github.com/garypwhite/>`_
 
 GSoC 2022 participants
 -----------------------

add.md

@@ -0,0 +1 @@
+dfadffd

augur/api/metrics/README.md

@@ -26,7 +26,8 @@ from augur.application.db.engine import engine
 4. Define any queries with the structure show below
 ```py
 repo_sql = s.sql.text(""" SELECT repo.repo_name FROM repo WHERE repo.repo_id = :repo_id """)
-results = pd.read_sql(repo_sql, engine, params={'repo_id': repo_id})
+with engine.connect() as conn:
+    results = pd.read_sql(repo_sql, conn, params={'repo_id': repo_id})
 ```
 5. Return either a pandas dataframe, dict, or json.
     - Note: If you return a pandas dataframe or dict it will be automatically converted into json

augur/api/metrics/commit.py

@@ -90,8 +90,9 @@ def committers(repo_group_id, repo_id=None, begin_date=None, end_date=None, peri
             """
         )
 
-    results = pd.read_sql(committersSQL, engine, params={'repo_id': repo_id, 
-        'repo_group_id': repo_group_id,'begin_date': begin_date, 'end_date': end_date, 'period':period})
+    with engine.connect() as conn:
+        results = pd.read_sql(committersSQL, conn, params={'repo_id': repo_id, 
+            'repo_group_id': repo_group_id,'begin_date': begin_date, 'end_date': end_date, 'period':period})
 
     return results
 
@@ -167,8 +168,9 @@ def annual_commit_count_ranked_by_new_repo_in_repo_group(repo_group_id, repo_id=
             ORDER BY YEAR ASC
         """.format(table, period))
 
-    results = pd.read_sql(cdRgNewrepRankedCommitsSQL, engine, params={'repo_id': repo_id, 
-        'repo_group_id': repo_group_id,'begin_date': begin_date, 'end_date': end_date})
+    with engine.connect() as conn:
+        results = pd.read_sql(cdRgNewrepRankedCommitsSQL, conn, params={'repo_id': repo_id, 
+            'repo_group_id': repo_group_id,'begin_date': begin_date, 'end_date': end_date})
     return results
 
 @register_metric()
@@ -265,8 +267,9 @@ def annual_commit_count_ranked_by_repo_in_repo_group(repo_group_id, repo_id=None
                 LIMIT 10
             """)
 
-    results = pd.read_sql(cdRgTpRankedCommitsSQL, engine, params={ "repo_group_id": repo_group_id,
-    "repo_id": repo_id})
+    with engine.connect() as conn:
+        results = pd.read_sql(cdRgTpRankedCommitsSQL, conn, params={ "repo_group_id": repo_group_id,
+        "repo_id": repo_id})
     return results
 
 @register_metric()
@@ -296,8 +299,9 @@ def top_committers(repo_group_id, repo_id=None, year=None, threshold=0.8):
                 ORDER BY patches DESC) a
         """)
 
-        results = pd.read_sql(total_commits_SQL, engine,
-                            params={'year': year, 'repo_group_id': repo_group_id})
+        with engine.connect() as conn:
+            results = pd.read_sql(total_commits_SQL, conn,
+                                params={'year': year, 'repo_group_id': repo_group_id})
     else:
         total_commits_SQL = s.sql.text("""
             SELECT SUM(patches)::int
@@ -308,8 +312,9 @@ def top_committers(repo_group_id, repo_id=None, year=None, threshold=0.8):
                 ORDER BY patches DESC) a
         """)
 
-        results = pd.read_sql(total_commits_SQL, engine,
-                            params={'year': year, 'repo_id': repo_id})
+        with engine.connect() as conn:
+            results = pd.read_sql(total_commits_SQL, conn,
+                                params={'year': year, 'repo_id': repo_id})
 
     if not results.iloc[0]['sum']:
         return pd.DataFrame()
@@ -334,8 +339,9 @@ def top_committers(repo_group_id, repo_id=None, year=None, threshold=0.8):
             ORDER BY commits DESC
         """)
 
-        results = pd.read_sql(committers_SQL, engine,
-                            params={'year': year, 'repo_group_id': repo_group_id})
+        with engine.connect() as conn:
+            results = pd.read_sql(committers_SQL, conn,
+                                params={'year': year, 'repo_group_id': repo_group_id})
     else:
         committers_SQL = s.sql.text("""
             SELECT
@@ -353,8 +359,9 @@ def top_committers(repo_group_id, repo_id=None, year=None, threshold=0.8):
             ORDER BY commits DESC
         """)
 
-        results = pd.read_sql(committers_SQL, engine,
-                            params={'year': year, 'repo_id': repo_id})
+        with engine.connect() as conn:
+            results = pd.read_sql(committers_SQL, conn,
+                                params={'year': year, 'repo_id': repo_id})
 
     cumsum = 0
     for i, row in results.iterrows():

augur/api/metrics/contributor.py

@@ -125,8 +125,9 @@ def contributors(repo_group_id, repo_id=None, period='day', begin_date=None, end
             ORDER BY total DESC
         """)
 
-        results = pd.read_sql(contributorsSQL, engine, params={'repo_id': repo_id, 'period': period,
-                                                                'begin_date': begin_date, 'end_date': end_date})
+        with engine.connect() as conn:
+            results = pd.read_sql(contributorsSQL, conn, params={'repo_id': repo_id, 'period': period,
+                                                                    'begin_date': begin_date, 'end_date': end_date})
     else:
         contributorsSQL = s.sql.text("""
         SELECT id::text                           AS user_id,
@@ -211,8 +212,9 @@ def contributors(repo_group_id, repo_id=None, period='day', begin_date=None, end
             ORDER BY total DESC
         """)
 
-        results = pd.read_sql(contributorsSQL, engine, params={'repo_group_id': repo_group_id, 'period': period,
-                                                            'begin_date': begin_date, 'end_date': end_date})
+        with engine.connect() as conn:
+            results = pd.read_sql(contributorsSQL, conn, params={'repo_group_id': repo_group_id, 'period': period,
+                                                                'begin_date': begin_date, 'end_date': end_date})
     return results
 
 @register_metric()
@@ -281,8 +283,9 @@ def contributors_new(repo_group_id, repo_id=None, period='day', begin_date=None,
             GROUP BY date, repo.repo_id, repo_name
             """)
 
-        results = pd.read_sql(contributorsNewSQL, engine, params={'repo_id': repo_id, 'period': period,
-                                                                'begin_date': begin_date, 'end_date': end_date})
+        with engine.connect() as conn:
+            results = pd.read_sql(contributorsNewSQL, conn, params={'repo_id': repo_id, 'period': period,
+                                                                    'begin_date': begin_date, 'end_date': end_date})
     else:
         contributorsNewSQL = s.sql.text("""
             SELECT date_trunc(:period, b.created_at::DATE) AS date, COUNT(id) AS new_contributors, repo.repo_id, repo_name
@@ -330,8 +333,9 @@ def contributors_new(repo_group_id, repo_id=None, period='day', begin_date=None,
             GROUP BY date, repo.repo_id, repo_name
             """)
 
-        results = pd.read_sql(contributorsNewSQL, engine, params={'repo_group_id': repo_group_id, 'period': period,
-                                                                   'begin_date': begin_date, 'end_date': end_date})
+        with engine.connect() as conn:
+            results = pd.read_sql(contributorsNewSQL, conn, params={'repo_group_id': repo_group_id, 'period': period,
+                                                                       'begin_date': begin_date, 'end_date': end_date})
     return results
 
 @register_metric()
@@ -351,7 +355,8 @@ def lines_changed_by_author(repo_group_id, repo_id=None):
             GROUP BY commits.repo_id, date_trunc('week', cmt_author_date::date), cmt_author_affiliation, cmt_author_email, repo_name
             ORDER BY date_trunc('week', cmt_author_date::date) ASC;
         """)
-        results = pd.read_sql(linesChangedByAuthorSQL, engine, params={"repo_id": repo_id})
+        with engine.connect() as conn:
+            results = pd.read_sql(linesChangedByAuthorSQL, conn, params={"repo_id": repo_id})
         return results
     else:
         linesChangedByAuthorSQL = s.sql.text("""
@@ -362,7 +367,8 @@ def lines_changed_by_author(repo_group_id, repo_id=None):
             GROUP BY repo_id, date_trunc('week', cmt_author_date::date), cmt_author_affiliation, cmt_author_email
             ORDER BY date_trunc('week', cmt_author_date::date) ASC;
         """)
-        results = pd.read_sql(linesChangedByAuthorSQL, engine, params={"repo_group_id": repo_group_id})
+        with engine.connect() as conn:
+            results = pd.read_sql(linesChangedByAuthorSQL, conn, params={"repo_group_id": repo_group_id})
         return results
 
 @register_metric()
@@ -420,8 +426,9 @@ def contributors_code_development(repo_group_id, repo_id=None, period='all', beg
             GROUP BY a.email, a.repo_id, repo_name
         """)
 
-        results = pd.read_sql(contributorsSQL, engine, params={'repo_id': repo_id, 'period': period,
-                                                                'begin_date': begin_date, 'end_date': end_date})
+        with engine.connect() as conn:
+            results = pd.read_sql(contributorsSQL, conn, params={'repo_id': repo_id, 'period': period,
+                                                                    'begin_date': begin_date, 'end_date': end_date})
     else:
         contributorsSQL = s.sql.text("""
             SELECT 
@@ -455,6 +462,7 @@ def contributors_code_development(repo_group_id, repo_id=None, period='all', beg
             ORDER BY commits desc, email
         """)
 
-        results = pd.read_sql(contributorsSQL, engine, params={'repo_group_id': repo_group_id, 'period': period,
-                                                                'begin_date': begin_date, 'end_date': end_date})
+        with engine.connect() as conn:
+            results = pd.read_sql(contributorsSQL, conn, params={'repo_group_id': repo_group_id, 'period': period,
+                                                                    'begin_date': begin_date, 'end_date': end_date})
     return results

augur/api/metrics/deps.py

@@ -6,6 +6,7 @@
 import sqlalchemy as s
 import pandas as pd
 from augur.api.util import register_metric
+import datetime
 
 from ..server import engine
 
@@ -45,7 +46,8 @@ def deps(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=No
             AND repo_dependencies.repo_id = :repo_id
             """)
 
-        results = pd.read_sql(depsSQL, engine)    	
+        with engine.connect() as conn:
+            results = pd.read_sql(depsSQL, conn)    	
 
     else:
 
@@ -69,7 +71,8 @@ def deps(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=No
             AND repo.repo_group_id = :repo_group_id
             """)
 
-        results = pd.read_sql(depsSQL, engine)
+        with engine.connect() as conn:
+            results = pd.read_sql(depsSQL, conn)
     return results
 
 

augur/api/metrics/insight.py

@@ -29,5 +29,6 @@ def top_insights(repo_group_id, num_repos=6):
             LIMIT :num_repos
         )
     """)
-    results = pd.read_sql(topInsightsSQL, engine, params={'repo_group_id': repo_group_id, 'num_repos': num_repos})
+    with engine.connect() as conn:
+        results = pd.read_sql(topInsightsSQL, conn, params={'repo_group_id': repo_group_id, 'num_repos': num_repos})
     return results