Update a copy of the ML patch with augur-new-dev

README.md

@@ -4,11 +4,7 @@
 
 [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml)
 
-branch | status
-   --- | ---
-  main | [![Build Status](https://travis-ci.com/chaoss/augur.svg?branch=main)](https://travis-ci.com/chaoss/augur)
-   dev | [![Build Status](https://travis-ci.com/chaoss/augur.svg?branch=dev)](https://travis-ci.com/chaoss/augur)
-
+[![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=augur-new)](https://hitsofcode.com/github/chaoss/augur/view?branch=augur-new)
 
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788)
 

augur/api/metrics/repo_meta.py

@@ -457,26 +457,50 @@ def license_coverage(repo_group_id, repo_id=None):
 
     if repo_id:
         license_declared_SQL = s.sql.text("""
-            SELECT a.name, b.total as total_files, a.licensed as license_declared_files, round(a.licensed/b.total::numeric, 3) as coverage
-            FROM (
-            SELECT packages.name as name, count(file_license_id) as licensed
-            FROM packages,
-                files_licenses,
-                packages_files,
-                repo
-            WHERE packages.name = repo.repo_name
-            and repo.repo_id = :repo_id
-            and packages.package_id = packages_files.package_id
-            and packages_files.file_id = files_licenses.file_id
-            GROUP BY packages.name) a, (SELECT packages.name as name, count(packages_files.file_id) as total
-            FROm packages, repo, packages_files
-            WHERE packages.name = repo.repo_name
-            and repo.repo_id = :repo_id
-            AND packages.package_id = packages_files.package_id
-            GROUP BY packages.name
-            )b
-            WHERE a.name = b.name
-            GROUP BY a.name, a.licensed, a.licensed, b.total
+        SELECT B.NAME,
+            b.total AS total_files,
+            COALESCE(A.licensed,0) AS license_declared_files,
+            round( COALESCE(A.licensed :: NUMERIC, 0) / b.total :: NUMERIC, 3 ) AS coverage 
+        FROM
+            (
+            SELECT
+                packages.NAME AS NAME,
+                COUNT ( packages_files.file_id ) AS total 
+            FROM
+                spdx.packages,
+                repo,
+                spdx.packages_files 
+            WHERE
+                packages.NAME = repo.repo_name 
+                AND repo.repo_id = :repo_id 
+                AND packages.package_id = packages_files.package_id 
+            GROUP BY
+                packages.NAME 
+            ) b 
+            left outer join 
+                (
+            SELECT
+                packages.NAME AS NAME,
+                COUNT ( * ) AS licensed 
+            FROM
+                spdx.packages,
+                spdx.files_licenses,
+                spdx.packages_files,
+                repo 
+            WHERE
+                packages.NAME = repo.repo_name 
+                AND repo.repo_id = :repo_id 
+                AND packages.package_id = packages_files.package_id 
+                AND packages_files.file_id = files_licenses.file_id 
+            GROUP BY
+                packages.NAME 
+            ) A
+        on 
+            A.NAME = b.NAME 
+        GROUP BY
+            B.NAME,
+            A.licensed,
+            b.total
         """)
     else:
         license_declared_SQL = s.sql.text("""

augur/api/routes/auggie.py

@@ -245,7 +245,7 @@
 def create_routes(server):
 
 
-    server.app.route('/auggie/get_user', methods=['POST'])
+    @server.app.route('/auggie/get_user', methods=['POST'])
     def get_auggie_user():
         # arg = [request.json]
         # response = server.transform(metrics.get_auggie_user, args=arg)
@@ -271,7 +271,7 @@ def get_auggie_user():
 
         return filteredUser
 
-    server.app.route('/auggie/update_tracking', methods=['POST'])
+    @server.app.route('/auggie/update_tracking', methods=['POST'])
     def update_auggie_user_tracking():
         # arg = [request.json]
         # response = server.transform(metrics.update_tracking, args=arg)
@@ -317,7 +317,7 @@ def update_auggie_user_tracking():
 
         return filtered_values
 
-    server.app.route('/auggie/slack_login', methods=['POST'])
+    @server.app.route('/auggie/slack_login', methods=['POST'])
     def slack_login():
         # arg = [request.json]
         # response = server.transform(metrics.slack_login, args=arg)

augur/api/routes/batch.py

@@ -18,7 +18,7 @@
 
 def create_routes(server):
 
-        server.app.route('/{}/batch'.format(AUGUR_API_VERSION), methods=['GET', 'POST'])
+        @server.app.route('/{}/batch'.format(AUGUR_API_VERSION), methods=['GET', 'POST'])
         def batch():
             """
             Execute multiple requests, submitted as a batch.
@@ -101,7 +101,7 @@ def batch():
         @apiDescription Returns metadata of batch requests
         POST JSON of API requests metadata
         """
-        server.app.route('/{}/batch/metadata'.format(AUGUR_API_VERSION), methods=['GET', 'POST'])
+        @server.app.route('/{}/batch/metadata'.format(AUGUR_API_VERSION), methods=['GET', 'POST'])
         def batch_metadata():
             """
             Returns endpoint metadata in batch format

augur/api/routes/collection_status.py

@@ -6,10 +6,11 @@
 
 AUGUR_API_VERSION = 'api/unstable'
 
+
 def create_routes(server):
 
-    server.app.route('/{}/collection_status/commits'.format(AUGUR_API_VERSION))
-    def commit_collection_status(): #TODO: make this name automatic - wrapper?
+    @server.app.route('/{}/collection_status/commits'.format(AUGUR_API_VERSION))
+    def commit_collection_status():  # TODO: make this name automatic - wrapper?
         commit_collection_sql = s.sql.text("""
             SELECT
                 repo_id,
@@ -34,13 +35,14 @@ def commit_collection_status(): #TODO: make this name automatic - wrapper?
                 repo_status = 'Complete'
         """)
         results = pd.read_sql(commit_collection_sql,  server.engine)
-        data = results.to_json(orient="records", date_format='iso', date_unit='ms')
+        data = results.to_json(
+            orient="records", date_format='iso', date_unit='ms')
         return Response(response=data,
                         status=200,
                         mimetype="application/json")
 
-    server.app.route('/{}/collection_status/issues'.format(AUGUR_API_VERSION))
-    def issue_collection_status(): #TODO: make this name automatic - wrapper?
+    @server.app.route('/{}/collection_status/issues'.format(AUGUR_API_VERSION))
+    def issue_collection_status():  # TODO: make this name automatic - wrapper?
         issue_collection_sql = s.sql.text("""
             SELECT
                 *
@@ -94,14 +96,15 @@ def issue_collection_status(): #TODO: make this name automatic - wrapper?
             WHERE d.issues_enabled = 'true';
         """)
         results = pd.read_sql(issue_collection_sql,  server.engine)
-        data = results.to_json(orient="records", date_format='iso', date_unit='ms')
+        data = results.to_json(
+            orient="records", date_format='iso', date_unit='ms')
         parsed_data = json.loads(data)
         return Response(response=data,
                         status=200,
                         mimetype="application/json")
 
-    server.app.route('/{}/collection_status/pull_requests'.format(AUGUR_API_VERSION))
-    def pull_request_collection_status(): #TODO: make this name automatic - wrapper?
+    @server.app.route('/{}/collection_status/pull_requests'.format(AUGUR_API_VERSION))
+    def pull_request_collection_status():  # TODO: make this name automatic - wrapper?
         pull_request_collection_sql = s.sql.text("""
             SELECT
                 *
@@ -163,7 +166,8 @@ def pull_request_collection_status(): #TODO: make this name automatic - wrapper?
                 ratio_abs;
         """)
         results = pd.read_sql(pull_request_collection_sql,  server.engine)
-        data = results.to_json(orient="records", date_format='iso', date_unit='ms')
+        data = results.to_json(
+            orient="records", date_format='iso', date_unit='ms')
         parsed_data = json.loads(data)
         return Response(response=data,
                         status=200,

augur/api/routes/config.py

@@ -5,9 +5,12 @@
 import logging
 import requests
 import json
+import os
 from flask import request, jsonify, Response
 import sqlalchemy as s
 
+# Disable the requirement for SSL by setting env["AUGUR_DEV"] = True
+development = os.getenv("AUGUR_DEV") or False
 
 from augur.application.db.models import Config
 from augur.application.db.session import DatabaseSession
@@ -31,9 +34,9 @@ def unsupported_method(error):
         return jsonify({"status": "Unsupported method"}), 405
 
 
-    @server.app.route(f"/{AUGUR_API_VERSION}/config/get", methods=['POST'])
+    @server.app.route(f"/{AUGUR_API_VERSION}/config/get", methods=['GET', 'POST'])
     def get_config():
-        if not request.is_secure:
+        if not development and not request.is_secure:
             return generate_upgrade_request()
 
         with DatabaseSession(logger) as session:
@@ -45,10 +48,10 @@ def get_config():
 
     @server.app.route(f"/{AUGUR_API_VERSION}/config/update", methods=['POST'])
     def update_config():
-        if not request.is_secure:
+        if not development and not request.is_secure:
             return generate_upgrade_request()
 
-        update_dict= request.get_json()
+        update_dict = request.get_json()
 
         with DatabaseSession(logger) as session:
 

augur/api/routes/contributor_reports.py

@@ -579,14 +579,16 @@ def filter_data(df, needed_columns, not_null_columns=[]):
         if all(x in needed_columns for x in not_null_columns):
 
             df = get_needed_columns(df, needed_columns)
-            df = remove_rows_with_null_values(df, not_null_columns)
+            #Use the pandas method bc the other method was erroring on boolean index.
+            #IM - 9/23/22
+            df = df.dropna(subset=not_null_columns)#remove_rows_with_null_values(df, not_null_columns)
 
             return df
         else:
             print("Developer error, not null columns should be a subset of needed columns")
             return df
 
-    server.app.route('/{}/contributor_reports/new_contributors_bar/'.format(AUGUR_API_VERSION), methods=["GET"])
+    @server.app.route('/{}/contributor_reports/new_contributors_bar/'.format(AUGUR_API_VERSION), methods=["GET"])
     def new_contributors_bar():
 
         repo_id, start_date, end_date, error = get_repo_id_start_date_and_end_date()
@@ -605,7 +607,9 @@ def new_contributors_bar():
         not_null_columns = ['cntrb_id', 'created_at', 'month', 'year', 'repo_id', 'repo_name', 'login', 'action',
                             'rank', 'yearmonth', 'new_contributors', 'quarter']
 
-        input_df = remove_rows_with_null_values(input_df, not_null_columns)
+        #Use the pandas method bc the other method was erroring on boolean index.
+        #IM - 9/23/22
+        input_df = input_df.dropna(subset=not_null_columns)#remove_rows_with_null_values(input_df, not_null_columns)
 
         if len(input_df) == 0:
             return Response(response="There is no data for this repo, in the database you are accessing",
@@ -755,7 +759,7 @@ def new_contributors_bar():
 
         return send_file(filename)
 
-    server.app.route('/{}/contributor_reports/new_contributors_stacked_bar/'.format(AUGUR_API_VERSION),
+    @server.app.route('/{}/contributor_reports/new_contributors_stacked_bar/'.format(AUGUR_API_VERSION),
                       methods=["GET"])
     def new_contributors_stacked_bar():
 
@@ -958,7 +962,7 @@ def new_contributors_stacked_bar():
 
         return send_file(filename)
 
-    server.app.route('/{}/contributor_reports/returning_contributors_pie_chart/'.format(AUGUR_API_VERSION),
+    @server.app.route('/{}/contributor_reports/returning_contributors_pie_chart/'.format(AUGUR_API_VERSION),
                       methods=["GET"])
     def returning_contributors_pie_chart():
 
@@ -1090,7 +1094,7 @@ def returning_contributors_pie_chart():
 
         return send_file(filename)
 
-    server.app.route('/{}/contributor_reports/returning_contributors_stacked_bar/'.format(AUGUR_API_VERSION),
+    @server.app.route('/{}/contributor_reports/returning_contributors_stacked_bar/'.format(AUGUR_API_VERSION),
                       methods=["GET"])
     def returning_contributors_stacked_bar():
 

augur/api/routes/manager.py

@@ -25,7 +25,7 @@ def create_routes(server):
 
     pass
 
-#     server.app.route('/{}/add-repos'.format(AUGUR_API_VERSION), methods=['POST'])
+#    @server.app.route('/{}/add-repos'.format(AUGUR_API_VERSION), methods=['POST'])
 #     def add_repos():
 #         """ returns list of successfully inserted repos and repos that caused an error
 #             adds repos belonging to any user or group to an existing augur repo group
@@ -68,7 +68,7 @@ def create_routes(server):
 #                         status=status_code,
 #                         mimetype="application/json")
 
-#     server.app.route('/{}/create-repo-group'.format(AUGUR_API_VERSION), methods=['POST'])
+#    @server.app.route('/{}/create-repo-group'.format(AUGUR_API_VERSION), methods=['POST'])
 #     def create_repo_group():
 #         if authenticate_request(server.augur_app, request):
 #             group = request.json['group']
@@ -103,7 +103,7 @@ def create_routes(server):
 #                         status=status_code, 
 #                         mimetype="application/json")
 
-#     server.app.route('/{}/import-org'.format(AUGUR_API_VERSION), methods=['POST'])
+#    @server.app.route('/{}/import-org'.format(AUGUR_API_VERSION), methods=['POST'])
 #     def add_repo_group():
 #         """ creates a new augur repo group and adds to it the given organization or user's repos
 #             takes an organization or user name 

augur/api/routes/metadata.py

@@ -16,7 +16,7 @@
 
 def create_routes(server):
 
-    server.app.route('/{}/metadata/repo_info'.format(AUGUR_API_VERSION), methods=["GET"])
+    @server.app.route('/{}/metadata/repo_info'.format(AUGUR_API_VERSION), methods=["GET"])
     def get_repo_info():
         repo_info_sql = s.sql.text("""
             SELECT
@@ -55,7 +55,7 @@ def get_repo_info():
                     status=200,
                     mimetype="application/json")
 
-    server.app.route('/{}/metadata/contributions_count'.format(AUGUR_API_VERSION), methods=["GET"])
+    @server.app.route('/{}/metadata/contributions_count'.format(AUGUR_API_VERSION), methods=["GET"])
     def contributions_count():
         repo_info_sql = s.sql.text("""
             select repo_git, count(*) as contributions from contributor_repo
@@ -69,7 +69,7 @@ def contributions_count():
                     status=200,
                     mimetype="application/json")
 
-    server.app.route('/{}/metadata/contributors_count'.format(AUGUR_API_VERSION), methods=["GET"])
+    @server.app.route('/{}/metadata/contributors_count'.format(AUGUR_API_VERSION), methods=["GET"])
     def contributors_count():
         repo_info_sql = s.sql.text("""
             select repo_git, count(distinct(cntrb_id)) as contributors from contributor_repo

augur/api/routes/nonstandard_metrics.py

@@ -15,15 +15,15 @@
 
 def create_routes(server):
 
-    server.app.route(f"/{AUGUR_API_VERSION}/<license_id>/<spdx_binary>/<repo_group_id>/<repo_id>/license-files")
+    @server.app.route(f"/{AUGUR_API_VERSION}/<license_id>/<spdx_binary>/<repo_group_id>/<repo_id>/license-files")
     def get_license_files(license_id, spdx_binary, repo_group_id, repo_id):
         arguments = [license_id, spdx_binary, repo_group_id, repo_id]
         license_files = server.transform(license_files, args=arguments)
         return Response(response=license_files,
                         status=200,
                         mimetype="application/json")
 
-    server.app.route(f"/{AUGUR_API_VERSION}/repo-groups/<repo_group_id>/top-insights")
+    @server.app.route(f"/{AUGUR_API_VERSION}/repo-groups/<repo_group_id>/top-insights")
     def top_insights(repo_group_id):
         data = server.transform(top_insights, args=[repo_group_id])
         return Response(response=data,