From fbf56e41d92dc9d6bbb0fac2ed67895c9423410e Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Wed, 3 Jul 2019 23:37:53 +0530 Subject: [PATCH 01/10] update augur_db tests Signed-off-by: Parth Sharma --- augur/datasources/augur_db/test_augur_db.py | 49 +++++++++---------- .../augur_db/test_augur_db_routes.py | 35 ++++++------- 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/augur/datasources/augur_db/test_augur_db.py b/augur/datasources/augur_db/test_augur_db.py index 51e5d60334..7f43be7a57 100644 --- a/augur/datasources/augur_db/test_augur_db.py +++ b/augur/datasources/augur_db/test_augur_db.py @@ -51,44 +51,44 @@ def test_code_changes_lines(augur_db): def test_issues_new(augur_db): #repo_id - assert augur_db.issues_new(23, 21403, period='year').iloc[0]['issues'] == 1 + assert augur_db.issues_new(23, 21403, period='year').iloc[0]['issues'] > 0 #repo_group_id - assert augur_db.issues_new(23, period='year').iloc[1]['issues'] == 1 + assert augur_db.issues_new(23, period='year').iloc[1]['issues'] > 0 #begin_date & end_date assert augur_db.issues_new(24, 21979, period='week', begin_date='2017', - end_date='2017-05').iloc[1]['issues'] == 4 + end_date='2017-05').iloc[1]['issues'] > 0 assert augur_db.issues_new(24, period='month', begin_date='2017-05', - end_date='2018').iloc[2]['issues'] == 2 + end_date='2018').iloc[2]['issues'] > 0 def test_issues_active(augur_db): # repo - assert augur_db.issues_active(22, 21326, period='year').iloc[0]['issues'] == 98 + assert augur_db.issues_active(22, 21326, period='year').iloc[0]['issues'] > 0 # repo_group - assert augur_db.issues_active(22, period='year').iloc[5]['issues'] == 20 + assert augur_db.issues_active(22, period='year').iloc[5]['issues'] > 0 # begin_date & end_date assert augur_db.issues_active(22, 21326, period='month', begin_date='2015', - end_date='2015-09').iloc[0]['issues'] == 32 + end_date='2015-09').iloc[0]['issues'] > 0 assert augur_db.issues_active(22, period='week', begin_date='2015-01', - end_date='2015-08-05') .iloc[0]['issues'] == 32 + end_date='2015-08-05') .iloc[0]['issues'] > 0 def test_issues_closed(augur_db): # repo - assert augur_db.issues_closed(24, 21681, period='year').iloc[0]['issues'] == 189 + assert augur_db.issues_closed(24, 21681, period='year').iloc[0]['issues'] > 0 #repo_group - assert augur_db.issues_closed(24, period='year').iloc[1]['issues'] == 97 + assert augur_db.issues_closed(24, period='year').iloc[1]['issues'] > 0 # begin_date & end_date assert augur_db.issues_closed(24, 21681, period='week', begin_date='2012', - end_date='2012-07').iloc[0]['issues'] == 10 + end_date='2012-07').iloc[0]['issues'] > 0 assert augur_db.issues_closed(24, period='month', begin_date='2012-05', - end_date='2012-08-15').iloc[0]['issues'] == 50 + end_date='2012-08-15').iloc[0]['issues'] > 0 def test_issue_duration(augur_db): # repo @@ -99,24 +99,24 @@ def test_issue_duration(augur_db): def test_issue_participants(augur_db): # repo - assert augur_db.issue_participants(23, 21403).iloc[0]['participants'] == 1 + assert augur_db.issue_participants(23, 21403).iloc[0]['participants'] > 0 # repo_group - assert augur_db.issue_participants(22).iloc[4]['participants'] == 4 + assert augur_db.issue_participants(22).iloc[4]['participants'] > 0 def test_issue_throughput(augur_db): # repo - assert augur_db.issue_throughput(20, 21009).iloc[0]['throughput'] == 0.263158 + assert augur_db.issue_throughput(20, 21009).iloc[0]['throughput'] >= 0 # repo_group - assert augur_db.issue_throughput(24).iloc[0]['throughput'] == 0.861896 + assert augur_db.issue_throughput(24).iloc[0]['throughput'] >= 0 def test_issue_backlog(augur_db): #repo_id - assert augur_db.issue_backlog(21, 21166).iloc[0]['issue_backlog'] == 4 + assert augur_db.issue_backlog(21, 21166).iloc[0]['issue_backlog'] > 0 #repo_group_id - assert augur_db.issue_backlog(21).iloc[2]['issue_backlog'] == 20 + assert augur_db.issue_backlog(21).iloc[2]['issue_backlog'] > 0 def test_issues_first_time_closed(augur_db): @@ -193,22 +193,22 @@ def test_contributors_new(augur_db): def test_open_issues_count(augur_db): # repo - assert augur_db.open_issues_count(22, 21326).iloc[0]['open_count'] == 1 + assert augur_db.open_issues_count(22, 21326).iloc[0]['open_count'] > 0 # repo_group - assert augur_db.open_issues_count(23).iloc[1]['open_count'] == 4 + assert augur_db.open_issues_count(23).iloc[1]['open_count'] > 0 def test_closed_issues_count(augur_db): # repo - assert augur_db.closed_issues_count(24, 21684).iloc[0]['closed_count'] == 3 + assert augur_db.closed_issues_count(24, 21684).iloc[0]['closed_count'] > 0 # repo_group - assert augur_db.closed_issues_count(20).iloc[0]['closed_count'] == 1 + assert augur_db.closed_issues_count(20).iloc[0]['closed_count'] > 0 def test_issues_open_age(augur_db): #repo group assert augur_db.issues_open_age(24).iloc[0]['open_date'] > 0 - # repo + # repo assert augur_db.issues_open_age(20,21000).iloc[0]['open_date'] > 0 def test_issues_closed_resolution_duration(augur_db): @@ -225,7 +225,7 @@ def test_lines_changed_by_author(augur_db): assert augur_db.lines_changed_by_author(20,21000).iloc[0].additions > 0 def test_annual_commit_count_ranked_by_new_repo_in_repo_group(augur_db): - assert augur_db.annual_commit_count_ranked_by_new_repo_in_repo_group(20, calendar_year=2019).iloc[0].net > 0 + assert augur_db.annual_commit_count_ranked_by_new_repo_in_repo_group(20, calendar_year=2019).iloc[0].net > 0 def test_cii_best_practices_badge(augur_db): # repo @@ -247,4 +247,3 @@ def test_issues_maintainer_response_duration(augur_db): assert augur_db.issues_maintainer_response_duration(20, 21000).iloc[0].average_days_comment > 0 assert augur_db.issues_maintainer_response_duration(20).iloc[0].average_days_comment > 0 assert augur_db.issues_maintainer_response_duration(20, 21000).iloc[0].average_days_comment > 0 - diff --git a/augur/datasources/augur_db/test_augur_db_routes.py b/augur/datasources/augur_db/test_augur_db_routes.py index fdce47556e..ac01dc5eae 100644 --- a/augur/datasources/augur_db/test_augur_db_routes.py +++ b/augur/datasources/augur_db/test_augur_db_routes.py @@ -83,21 +83,21 @@ def test_issues_new_by_repo(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['issues'] >= 1 + assert data[0]['issues'] > 0 def test_issues_active_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issues-active') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['issues'] >= 1 + assert data[0]['issues'] > 0 def test_issues_active_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21339/issues-active') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['issues'] >= 1 + assert data[0]['issues'] > 0 def test_issues_closed_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/24/issues-closed') @@ -118,42 +118,40 @@ def test_issue_duration_by_group(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['duration'] == '60 days 19:41:27.000000000' def test_issue_duration_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/repos/21009/issue-duration') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['duration'] == '60 days 19:41:27.000000000' def test_issue_participants_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issue-participants') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['participants'] == 1 + assert data[0]['participants'] > 0 def test_issue_participants_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21403/issue-participants') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['participants'] == 1 + assert data[0]['participants'] > 0 def test_issue_throughput_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/issue-throughput') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['throughput'] == 0.263158 + assert data[0]['throughput'] >= 0 def test_issue_throughput_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/24/repos/21682/issue-throughput') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['throughput'] == 0.861896 + assert data[0]['throughput'] >= 0 def test_issue_backlog_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issue-backlog') @@ -216,14 +214,14 @@ def test_sub_projects_by_group(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["sub_protject_count"] > 0 + assert data[0]["sub_project_count"] > 0 def test_sub_projects_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/24/repos/21477/sub-projects') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["sub_protject_count"] > 0 + assert data[0]["sub_project_count"] > 0 def test_contributors_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/contributors') @@ -260,28 +258,28 @@ def test_open_issues_count_by_group(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['open_count'] == 1 + assert data[0]['open_count'] > 0 def test_open_issues_count_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/22/repos/21326/open-issues-count') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['open_count'] == 1 + assert data[0]['open_count'] > 0 def test_closed_issues_count_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/closed-issues-count') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['closed_count'] == 1 + assert data[0]['closed_count'] > 0 def test_closed_issues_count_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/22/repos/21684/closed-issues-count') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]['closed_count'] == 3 + assert data[0]['closed_count'] > 0 def test_issues_open_age_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/issues-open-age/') @@ -317,7 +315,7 @@ def test_annual_commit_count_ranked_by_new_repo_in_repo_group(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["net"] >= 0 + assert data[0]["net"] >= 0 def test_issues_maintainer_response_duration_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/repos/21000/issues-maintainer-response-duration/') @@ -339,14 +337,12 @@ def test_cii_best_practices_badge_by_group(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["badge_level"] == 'passing' def test_cii_best_practices_badge_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/21/repos/21252/cii-best-practices-badge') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["badge_level"] == 'in_progress' def test_languages_by_group(augur_db_routes): # TODO need data @@ -361,12 +357,9 @@ def test_license_declared_by_group(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["license"] == 'Apache-2.0' def test_license_declared_by_repo(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/21/repos/21252/license-declared') data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["license"] == 'Apache-2.0' - From dd5227f76726652a2ac302bf23e08dcfdf342632 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Wed, 3 Jul 2019 23:42:22 +0530 Subject: [PATCH 02/10] fix key errors in gh_repo_info_worker Signed-off-by: Parth Sharma --- .../gh_repo_info_worker/gh_repo_info_worker/worker.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/workers/gh_repo_info_worker/gh_repo_info_worker/worker.py b/workers/gh_repo_info_worker/gh_repo_info_worker/worker.py index b7b6240e91..074cde6ae8 100644 --- a/workers/gh_repo_info_worker/gh_repo_info_worker/worker.py +++ b/workers/gh_repo_info_worker/gh_repo_info_worker/worker.py @@ -140,8 +140,11 @@ def collect(self, repos=None): for _, row in repos.iterrows(): owner, repo = self.get_owner_repo(row['repo_git']) + print(f'Querying: {owner}/{repo}') self.query_repo_info(row['repo_id'], owner, repo) + print(f'Added repo info for {self.results_counter} repos') + def get_owner_repo(self, git_url): split = git_url.split('/') @@ -206,16 +209,16 @@ def query_repo_info(self, repo_id, owner, repo): 'repo_id': repo_id, 'last_updated': j['updatedAt'], 'issues_enabled': j['hasIssuesEnabled'], - 'open_issues': j['issues']['totalCount'], + 'open_issues': j['issues']['totalCount'] if j['issues'] else None, 'pull_requests_enabled': None, 'wiki_enabled': j['hasWikiEnabled'], 'pages_enabled': None, 'fork_count': j['forkCount'], - 'default_branch': j['defaultBranchRef']['name'], - 'watchers_count': j['watchers']['totalCount'], + 'default_branch': j['defaultBranchRef']['name'] if j['defaultBranchRef'] else None, + 'watchers_count': j['watchers']['totalCount'] if j['watchers'] else None, 'UUID': None, 'license': j['licenseInfo']['name'] if j['licenseInfo'] else None, - 'stars_count': j['stargazers']['totalCount'], + 'stars_count': j['stargazers']['totalCount'] if j['stargazers'] else None, 'committers_count': None, 'issue_contributors_count': None, 'changelog_file': None, From d3f5e46b063fe3d2a7142a584c29a2795fc469fb Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Thu, 4 Jul 2019 18:56:30 +0530 Subject: [PATCH 03/10] add 'forks' metric implementation Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 48 ++++++++++++++++++-- augur/datasources/augur_db/routes.py | 61 ++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index f5272ba966..2949e99521 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1069,14 +1069,14 @@ def issues_open_age(self, repo_group_id, repo_id=None, period='day', begin_date= :param repo_id: The repository's repo_id, defaults to None :return: DataFrame of age of open issues. """ - + if not begin_date: begin_date = '1970-1-1 00:00:01' if not end_date: end_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') openAgeSQL = None - + if not repo_id: openAgeSQL = s.sql.text(""" SELECT repo.repo_id, repo_name, issue_id, date_trunc(:period, issues.created_at ) as date, EXTRACT(DAY FROM NOW() - issues.created_at) AS open_date @@ -1161,9 +1161,9 @@ def issues_closed_resolution_duration(self, repo_group_id, repo_id=None, period= GROUP BY repo.repo_id, repo.repo_name, gh_issue_number, issue_title, issues.created_at, issues.closed_at, DIFFDATE ORDER BY gh_issue_number """) - + results = pd.read_sql(issueSQL, self.db, - params={'repo_id': repo_id, + params={'repo_id': repo_id, 'repo_group_id': repo_group_id, 'period': period, 'begin_date':begin_date, 'end_date':end_date}) @@ -1202,6 +1202,46 @@ def cii_best_practices_badge(self, repo_group_id, repo_id=None): results = pd.read_sql(cii_best_practices_badge_SQL, self.db, params={'repo_id': repo_id}) return results + @annotate(tag='forks') + def forks(self, repo_group_id, repo_id=None): + """ + Returns a time series of the fork count + + :param repo_group_id: The repository's repo_group_id + :param repo_id: The repository's repo_id, defaults to None + :return: Time series of fork count + """ + if not repo_id: + forks_SQL = s.sql.text(""" + SELECT + repo_info.repo_id, + repo_name, + repo_info.data_collection_date as date, + fork_count AS forks + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id IN + (SELECT repo_id FROM repo + WHERE repo_group_id = :repo_group_id) + ORDER BY repo_info.repo_id, date + """) + + results = pd.read_sql(forks_SQL, self.db, params={'repo_group_id': repo_group_id}) + return results + + else: + forks_SQL = s.sql.text(""" + SELECT + repo_name, + repo_info.data_collection_date as date, + fork_count AS forks + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id = :repo_id + ORDER BY date + """) + + results = pd.read_sql(forks_SQL, self.db, params={'repo_id': repo_id}) + return results + @annotate(tag='languages') def languages(self, repo_group_id, repo_id=None): """Returns the implementation languages diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index 7929c5f636..bb15ce272a 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -1392,6 +1392,67 @@ def get_repos_for_dosocs(): """ server.addRepoMetric(augur_db.cii_best_practices_badge, 'cii-best-practices-badge') + """ + @api {get} /repo-groups/:repo_group_id/forks Forks (Repo Group) + @apiName forks-repo-group + @apiGroup Risk + @apiDescription A time series of fork count. + CHAOSS Metric Definition + @apiParam {string} repo_group_id Repository Group ID + @apiSuccessExample {json} Success-Response: + [ + { + "repo_id": 21036, + "repo_name": "jquery-ujs", + "date": "2019-07-03T23:26:42.000Z", + "forks": 519 + }, + { + "repo_id": 21036, + "repo_name": "jquery-ujs", + "date": "2019-07-04T16:39:39.000Z", + "forks": 519 + }, + { + "repo_id": 21039, + "repo_name": "rails_xss", + "date": "2019-07-03T23:26:22.000Z", + "forks": 20 + }, + { + "repo_id": 21039, + "repo_name": "rails_xss", + "date": "2019-07-04T16:39:20.000Z", + "forks": 20 + } + ] + """ + server.addRepoGroupMetric(augur_db.forks, 'forks') + + """ + @api {get} /repo-groups/:repo_group_id/repos/:repo_id/forks Forks (Repo) + @apiName forks-repo + @apiGroup Risk + @apiDescription A time series of fork count. + CHAOSS Metric Definition + @apiParam {string} repo_group_id Repository Group ID. + @apiParam {string} repo_id Repository ID. + @apiSuccessExample {json} Success-Response: + [ + { + "repo_name": "graphiql", + "date": "2019-07-03T23:27:42.000Z", + "forks": 843 + }, + { + "repo_name": "graphiql", + "date": "2019-07-04T16:40:44.000Z", + "forks": 844 + } + ] + """ + server.addRepoMetric(augur_db.forks, 'forks') + """ @api {get} /repo-groups/:repo_group_id/languages Languages (Repo Group) @apiName languages-repo-group From 6adb9f9cb5f0cfc6a8b8c7a984249b2ea2352c04 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Thu, 4 Jul 2019 19:03:38 +0530 Subject: [PATCH 04/10] add 'fork_count' metric implementation Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 35 ++++++++++++++++++++++ augur/datasources/augur_db/routes.py | 41 ++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index 2949e99521..4866f471d6 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1242,6 +1242,41 @@ def forks(self, repo_group_id, repo_id=None): results = pd.read_sql(forks_SQL, self.db, params={'repo_id': repo_id}) return results + @annotate(tag='fork-count') + def fork_count(self, repo_group_id, repo_id=None): + """ + Returns the latest fork count + + :param repo_group_id: The repository's repo_group_id + :param repo_id: The repository's repo_id, defaults to None + :return: Fork count + """ + if not repo_id: + fork_count_SQL = s.sql.text(""" + SELECT a.repo_id, repo_name, a.fork_count AS forks + FROM repo_info a LEFT JOIN repo_info b + ON (a.repo_id = b.repo_id AND a.repo_info_id < b.repo_info_id), repo + WHERE b.repo_info_id IS NULL + AND a.repo_id = repo.repo_id + AND a.repo_id IN + (SELECT repo_id FROM repo + WHERE repo_group_id = :repo_group_id) + """) + + results = pd.read_sql(fork_count_SQL, self.db, params={'repo_group_id': repo_group_id}) + return results + else: + fork_count_SQL = s.sql.text(""" + SELECT repo_name, fork_count AS forks + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id = :repo_id + ORDER BY repo_info.data_collection_date DESC + LIMIT 1 + """) + + results = pd.read_sql(fork_count_SQL, self.db, params={'repo_id': repo_id}) + return results + @annotate(tag='languages') def languages(self, repo_group_id, repo_id=None): """Returns the implementation languages diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index bb15ce272a..2c5cd430de 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -1453,6 +1453,47 @@ def get_repos_for_dosocs(): """ server.addRepoMetric(augur_db.forks, 'forks') + """ + @api {get} /repo-groups/:repo_group_id/fork-count Fork Count (Repo Group) + @apiName fork-count-repo-group + @apiGroup Risk + @apiDescription Fork count. + CHAOSS Metric Definition + @apiParam {string} repo_group_id Repository Group ID + @apiSuccessExample {json} Success-Response: + [ + { + "repo_id": 21364, + "repo_name": "irs_process_scripts", + "forks": 4 + }, + { + "repo_id": 21420, + "repo_name": "ruby-coffee-script", + "forks": 54 + } + ] + """ + server.addRepoGroupMetric(augur_db.fork_count, 'fork-count') + + """ + @api {get} /repo-groups/:repo_group_id/repos/:repo_id/fork-count Fork Count (Repo) + @apiName fork-count-repo + @apiGroup Risk + @apiDescription Fork count. + CHAOSS Metric Definition + @apiParam {string} repo_group_id Repository Group ID. + @apiParam {string} repo_id Repository ID. + @apiSuccessExample {json} Success-Response: + [ + { + "repo_name": "graphiql", + "forks": 844 + } + ] + """ + server.addRepoMetric(augur_db.fork_count, 'fork-count') + """ @api {get} /repo-groups/:repo_group_id/languages Languages (Repo Group) @apiName languages-repo-group From 45ee33b339ee056e39d76e7b035015c3ef05e806 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Fri, 5 Jul 2019 15:31:21 +0530 Subject: [PATCH 05/10] rename 'concat' in 'get_repos_for_dosocs' to 'path' Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index 4866f471d6..e96ea6244f 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1697,7 +1697,7 @@ def get_repo(self, owner, repo): def get_repos_for_dosocs(self): """ Returns a list of repos along with their repo_id & path """ get_repos_for_dosocs_SQL = s.sql.text(""" - SELECT b.repo_id, CONCAT(a.value || b.repo_group_id || chr(47) || b.repo_path || b.repo_name) + SELECT b.repo_id, CONCAT(a.value || b.repo_group_id || chr(47) || b.repo_path || b.repo_name) AS path FROM settings a, repo b WHERE a.setting='repo_directory' """) From dd14f1cb14a5326be5b88bcae03ba9f7b1fcfe9c Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Fri, 5 Jul 2019 15:31:57 +0530 Subject: [PATCH 06/10] add function test for 'get_repos_for_dosocs' Signed-off-by: Parth Sharma --- augur/datasources/augur_db/test_augur_db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/augur/datasources/augur_db/test_augur_db.py b/augur/datasources/augur_db/test_augur_db.py index 7f43be7a57..5b43034b86 100644 --- a/augur/datasources/augur_db/test_augur_db.py +++ b/augur/datasources/augur_db/test_augur_db.py @@ -247,3 +247,7 @@ def test_issues_maintainer_response_duration(augur_db): assert augur_db.issues_maintainer_response_duration(20, 21000).iloc[0].average_days_comment > 0 assert augur_db.issues_maintainer_response_duration(20).iloc[0].average_days_comment > 0 assert augur_db.issues_maintainer_response_duration(20, 21000).iloc[0].average_days_comment > 0 + +def test_get_repos_for_dosocs(augur_db): + assert augur_db.get_repos_for_dosocs().isin( + ['/home/sean/git-repos/23/github.com/rails/rails-dom-testing']).any().any() \ No newline at end of file From c1733494f704eb10b147e8f5b5410ee1a6bea367 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Fri, 5 Jul 2019 16:07:42 +0530 Subject: [PATCH 07/10] add 'stars' metric implementation Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 44 ++++++++++++++++++ augur/datasources/augur_db/routes.py | 63 ++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index e96ea6244f..a99b4c3f46 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1405,6 +1405,50 @@ def issues_maintainer_response_duration(self, repo_group_id, repo_id=None, begin return results + ##################################### + ### VALUE ### + ##################################### + + @annotate(tag='stars') + def stars(self, repo_group_id, repo_id=None): + """ + Returns a time series of the stars count + + :param repo_group_id: The repository's repo_group_id + :param repo_id: The repository's repo_id, defaults to None + :return: Time series of stars count + """ + if not repo_id: + stars_SQL = s.sql.text(""" + SELECT + repo_info.repo_id, + repo_name, + repo_info.data_collection_date as date, + stars_count AS stars + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id IN + (SELECT repo_id FROM repo + WHERE repo_group_id = :repo_group_id) + ORDER BY repo_info.repo_id, date + """) + + results = pd.read_sql(stars_SQL, self.db, params={'repo_group_id': repo_group_id}) + return results + + else: + stars_SQL = s.sql.text(""" + SELECT + repo_name, + repo_info.data_collection_date as date, + stars_count AS stars + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id = :repo_id + ORDER BY date + """) + + results = pd.read_sql(stars_SQL, self.db, params={'repo_id': repo_id}) + return results + ##################################### ### EXPERIMENTAL ### ##################################### diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index 2c5cd430de..8e5c9b89d1 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -1573,6 +1573,69 @@ def get_repos_for_dosocs(): """ server.addRepoMetric(augur_db.license_declared, 'license-declared') + ##################################### + ### VALUE ### + ##################################### + + """ + @api {get} /repo-groups/:repo_group_id/stars Stars (Repo Group) + @apiName stars-repo-group + @apiGroup Value + @apiDescription A time series of stars count. + @apiParam {string} repo_group_id Repository Group ID + @apiSuccessExample {json} Success-Response: + [ + { + "repo_id": 21491, + "repo_name": "commons-io", + "date": "2019-07-03T23:23:36.000Z", + "stars": 600 + }, + { + "repo_id": 21491, + "repo_name": "commons-io", + "date": "2019-07-04T16:36:27.000Z", + "stars": 601 + }, + { + "repo_id": 21524, + "repo_name": "maven", + "date": "2019-07-03T23:21:14.000Z", + "stars": 1730 + }, + { + "repo_id": 21524, + "repo_name": "maven", + "date": "2019-07-04T16:34:04.000Z", + "stars": 1733 + } + ] + """ + server.addRepoGroupMetric(augur_db.stars, 'stars') + + """ + @api {get} /repo-groups/:repo_group_id/repos/:repo_id/stars Stars (Repo) + @apiName stars-repo + @apiGroup Value + @apiDescription A time series of stars count. + @apiParam {string} repo_group_id Repository Group ID. + @apiParam {string} repo_id Repository ID. + @apiSuccessExample {json} Success-Response: + [ + { + "repo_name": "graphiql", + "date": "2019-07-03T23:27:42.000Z", + "stars": 8652 + }, + { + "repo_name": "graphiql", + "date": "2019-07-04T16:40:44.000Z", + "stars": 8653 + } + ] + """ + server.addRepoMetric(augur_db.stars, 'stars') + ##################################### ### EXPERIMENTAL ### ##################################### From 07db3743f4b9fae11881d59b293af038e71e94c4 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Fri, 5 Jul 2019 16:31:29 +0530 Subject: [PATCH 08/10] add 'stars-count' metric implementation Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 35 +++++++++++++++++++++++ augur/datasources/augur_db/routes.py | 39 ++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index a99b4c3f46..28c2036d2c 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1449,6 +1449,41 @@ def stars(self, repo_group_id, repo_id=None): results = pd.read_sql(stars_SQL, self.db, params={'repo_id': repo_id}) return results + @annotate(tag='stars-count') + def stars_count(self, repo_group_id, repo_id=None): + """ + Returns the latest stars count + + :param repo_group_id: The repository's repo_group_id + :param repo_id: The repository's repo_id, defaults to None + :return: stars count + """ + if not repo_id: + stars_count_SQL = s.sql.text(""" + SELECT a.repo_id, repo_name, a.stars_count AS stars + FROM repo_info a LEFT JOIN repo_info b + ON (a.repo_id = b.repo_id AND a.repo_info_id < b.repo_info_id), repo + WHERE b.repo_info_id IS NULL + AND a.repo_id = repo.repo_id + AND a.repo_id IN + (SELECT repo_id FROM repo + WHERE repo_group_id = :repo_group_id) + """) + + results = pd.read_sql(stars_count_SQL, self.db, params={'repo_group_id': repo_group_id}) + return results + else: + stars_count_SQL = s.sql.text(""" + SELECT repo_name, stars_count AS stars + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id = :repo_id + ORDER BY repo_info.data_collection_date DESC + LIMIT 1 + """) + + results = pd.read_sql(stars_count_SQL, self.db, params={'repo_id': repo_id}) + return results + ##################################### ### EXPERIMENTAL ### ##################################### diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index 8e5c9b89d1..dd57b8cffc 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -1636,6 +1636,45 @@ def get_repos_for_dosocs(): """ server.addRepoMetric(augur_db.stars, 'stars') + """ + @api {get} /repo-groups/:repo_group_id/stars-count Stars Count (Repo Group) + @apiName stars-count-repo-group + @apiGroup Value + @apiDescription Stars count. + @apiParam {string} repo_group_id Repository Group ID + @apiSuccessExample {json} Success-Response: + [ + { + "repo_id": 21364, + "repo_name": "irs_process_scripts", + "stars": 20 + }, + { + "repo_id": 21420, + "repo_name": "ruby-coffee-script", + "stars": 19 + } + ] + """ + server.addRepoGroupMetric(augur_db.stars_count, 'stars-count') + + """ + @api {get} /repo-groups/:repo_group_id/repos/:repo_id/stars-count Stars Count (Repo) + @apiName stars-count-repo + @apiGroup Value + @apiDescription Stars count. + @apiParam {string} repo_group_id Repository Group ID. + @apiParam {string} repo_id Repository ID. + @apiSuccessExample {json} Success-Response: + [ + { + "repo_name": "graphiql", + "stars": 8653 + } + ] + """ + server.addRepoMetric(augur_db.stars_count, 'stars-count') + ##################################### ### EXPERIMENTAL ### ##################################### From 3db1f1f92f5b68cff3d0639591211a9ac250d318 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Fri, 5 Jul 2019 16:42:43 +0530 Subject: [PATCH 09/10] add 'watchers' metric implementation Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 40 +++++++++++++++++ augur/datasources/augur_db/routes.py | 59 ++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index 28c2036d2c..b908a397e8 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1484,6 +1484,46 @@ def stars_count(self, repo_group_id, repo_id=None): results = pd.read_sql(stars_count_SQL, self.db, params={'repo_id': repo_id}) return results + @annotate(tag='watchers') + def watchers(self, repo_group_id, repo_id=None): + """ + Returns a time series of the watchers count + + :param repo_group_id: The repository's repo_group_id + :param repo_id: The repository's repo_id, defaults to None + :return: Time series of watchers count + """ + if not repo_id: + watchers_SQL = s.sql.text(""" + SELECT + repo_info.repo_id, + repo_name, + repo_info.data_collection_date as date, + watchers_count AS watchers + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id IN + (SELECT repo_id FROM repo + WHERE repo_group_id = :repo_group_id) + ORDER BY repo_info.repo_id, date + """) + + results = pd.read_sql(watchers_SQL, self.db, params={'repo_group_id': repo_group_id}) + return results + + else: + watchers_SQL = s.sql.text(""" + SELECT + repo_name, + repo_info.data_collection_date as date, + watchers_count AS watchers + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id = :repo_id + ORDER BY date + """) + + results = pd.read_sql(watchers_SQL, self.db, params={'repo_id': repo_id}) + return results + ##################################### ### EXPERIMENTAL ### ##################################### diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index dd57b8cffc..44d490c479 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -1675,6 +1675,65 @@ def get_repos_for_dosocs(): """ server.addRepoMetric(augur_db.stars_count, 'stars-count') + """ + @api {get} /repo-groups/:repo_group_id/watchers Watchers (Repo Group) + @apiName watchers-repo-group + @apiGroup Value + @apiDescription A time series of watchers count. + @apiParam {string} repo_group_id Repository Group ID + @apiSuccessExample {json} Success-Response: + [ + { + "repo_id": 21036, + "repo_name": "jquery-ujs", + "date": "2019-07-03T23:26:42.000Z", + "watchers": 60 + }, + { + "repo_id": 21036, + "repo_name": "jquery-ujs", + "date": "2019-07-04T16:39:39.000Z", + "watchers": 60 + }, + { + "repo_id": 21039, + "repo_name": "rails_xss", + "date": "2019-07-03T23:26:22.000Z", + "watchers": 19 + }, + { + "repo_id": 21039, + "repo_name": "rails_xss", + "date": "2019-07-04T16:39:20.000Z", + "watchers": 20 + } + ] + """ + server.addRepoGroupMetric(augur_db.watchers, 'watchers') + + """ + @api {get} /repo-groups/:repo_group_id/repos/:repo_id/watchers Watchers (Repo) + @apiName watchers-repo + @apiGroup Value + @apiDescription A time series of watchers count. + @apiParam {string} repo_group_id Repository Group ID. + @apiParam {string} repo_id Repository ID. + @apiSuccessExample {json} Success-Response: + [ + { + "repo_name": "airflow", + "date": "2019-07-03T23:22:26.000Z", + "watchers": 649 + }, + { + "repo_name": "airflow", + "date": "2019-07-04T16:35:16.000Z", + "watchers": 647 + } + ] + """ + server.addRepoMetric(augur_db.watchers, 'watchers') + ##################################### ### EXPERIMENTAL ### ##################################### From 4c11afd0af3a0dd6cb5f4e5ceb8a05a0bc9abc6b Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Fri, 5 Jul 2019 16:55:08 +0530 Subject: [PATCH 10/10] add 'watchers-count' metric implementation Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 35 +++++++++++++++++++++++ augur/datasources/augur_db/routes.py | 39 ++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index b908a397e8..fbe3b1279b 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -1524,6 +1524,41 @@ def watchers(self, repo_group_id, repo_id=None): results = pd.read_sql(watchers_SQL, self.db, params={'repo_id': repo_id}) return results + @annotate(tag='watchers-count') + def watchers_count(self, repo_group_id, repo_id=None): + """ + Returns the latest watchers count + + :param repo_group_id: The repository's repo_group_id + :param repo_id: The repository's repo_id, defaults to None + :return: watchers count + """ + if not repo_id: + watchers_count_SQL = s.sql.text(""" + SELECT a.repo_id, repo_name, a.watchers_count AS watchers + FROM repo_info a LEFT JOIN repo_info b + ON (a.repo_id = b.repo_id AND a.repo_info_id < b.repo_info_id), repo + WHERE b.repo_info_id IS NULL + AND a.repo_id = repo.repo_id + AND a.repo_id IN + (SELECT repo_id FROM repo + WHERE repo_group_id = :repo_group_id) + """) + + results = pd.read_sql(watchers_count_SQL, self.db, params={'repo_group_id': repo_group_id}) + return results + else: + watchers_count_SQL = s.sql.text(""" + SELECT repo_name, watchers_count AS watchers + FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id + WHERE repo_info.repo_id = :repo_id + ORDER BY repo_info.data_collection_date DESC + LIMIT 1 + """) + + results = pd.read_sql(watchers_count_SQL, self.db, params={'repo_id': repo_id}) + return results + ##################################### ### EXPERIMENTAL ### ##################################### diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index 44d490c479..4f35bfe749 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -1734,6 +1734,45 @@ def get_repos_for_dosocs(): """ server.addRepoMetric(augur_db.watchers, 'watchers') + """ + @api {get} /repo-groups/:repo_group_id/watchers-count Watchers Count (Repo Group) + @apiName watchers-count-repo-group + @apiGroup Value + @apiDescription Watchers count. + @apiParam {string} repo_group_id Repository Group ID + @apiSuccessExample {json} Success-Response: + [ + { + "repo_id": 21039, + "repo_name": "rails_xss", + "watchers": 20 + }, + { + "repo_id": 21036, + "repo_name": "jquery-ujs", + "watchers": 60 + } + ] + """ + server.addRepoGroupMetric(augur_db.watchers_count, 'watchers-count') + + """ + @api {get} /repo-groups/:repo_group_id/repos/:repo_id/watchers-count watchers Count (Repo) + @apiName watchers-count-repo + @apiGroup Value + @apiDescription Watchers count. + @apiParam {string} repo_group_id Repository Group ID. + @apiParam {string} repo_id Repository ID. + @apiSuccessExample {json} Success-Response: + [ + { + "repo_name": "airflow", + "watchers": 649 + } + ] + """ + server.addRepoMetric(augur_db.watchers_count, 'watchers-count') + ##################################### ### EXPERIMENTAL ### #####################################