Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/test'
Browse files Browse the repository at this point in the history
  • Loading branch information
@sgoggins
sgoggins committed Apr 26, 2021
2 parents 36b0dd3 + c4ef918 commit eef9a54
Show file tree
Hide file tree
Showing 28 changed files with 7,860 additions and 5,131 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ rebuild:
@ scripts/control/rebuild.sh prod

rebuild-dev:
@ scripts/control/rebuild.sh dev
@ scripts/control/rebuild-backend.sh dev


#
Expand Down
73 changes: 51 additions & 22 deletions augur/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,35 @@
import os
import json
import logging
# generate random integer values
from random import seed
from random import randint
# seed random number generator
# DO NOT create a seed.
# Actual randomness is generated without a seed
# for this use case.

contributor_breadth_worker_p = randint(48000, 48500)
facade_worker_p = randint(48501, 49000)
insight_worker_p = randint(49002, 49500)
metric_status_worker_p = randint(49501, 50000)
pull_request_worker_p = randint(50001, 50500)
repo_info_worker_p = randint(50501, 51000)
value_worker_p = randint(51002, 51500)
contributor_worker_p = randint(52000, 52500)
message_insights_worker_p = randint(53000, 53499)
pull_request_analysis_worker_p = randint(54000, 54500)
discourse_analysis_worker_p = randint(54500, 54999)
message_insights_worker_p = randint(55000, 55499)
clustering_worker_p = randint(57000, 57499)
github_worker_p = randint(57500, 58000)
linux_badge_worker_p = randint(47000,47499)
gitlab_issues_worker_p = randint(47500,47999)
release_worker_p = randint(56000, 56499)
gitlab_merge_request_worker_p = randint(56500, 56999)
main_port = randint(5001,5300)



from augur.logging import ROOT_AUGUR_DIRECTORY

Expand Down Expand Up @@ -198,23 +227,23 @@
},
"Workers": {
"contributor_breadth_worker": {
"port": 50003,
"port": contributor_breadth_worker_p,
"switch": 0,
"workers": 1
},
"facade_worker": {
"port": 50100,
"port": facade_worker_p,
"repo_directory": "repos/",
"switch": 1,
"workers": 1
},
"github_worker": {
"port": 50200,
"port": github_worker_p,
"switch": 1,
"workers": 1
},
"insight_worker": {
"port": 50300,
"port": insight_worker_p,
"metrics": {"issues-new": "issues", "code-changes": "commit_count", "code-changes-lines": "added",
"reviews": "pull_requests", "contributors-new": "new_contributors"},
"confidence_interval": 95,
Expand All @@ -225,89 +254,89 @@
"anomaly_days": 14
},
"linux_badge_worker": {
"port": 50400,
"port": linux_badge_worker_p ,
"switch": 1,
"workers": 1
},
"metric_status_worker": {
"port": 50500,
"port": metric_status_worker_p,
"switch": 0,
"workers": 1
},
"pull_request_worker": {
"port": 50600,
"port": pull_request_worker_p,
"switch": 1,
"workers": 1
},
"repo_info_worker": {
"port": 50700,
"port": repo_info_worker_p,
"switch": 1,
"workers": 1
},
"value_worker": {
"port": 50800,
"port": value_worker_p,
"scc_bin": "scc",
"switch": 0,
"workers": 1
},
"contributor_worker": {
"port": 50900,
"port": contributor_worker_p,
"switch": 0,
"workers": 1
},
"gitlab_issues_worker": {
"port": 51000,
"port": gitlab_issues_worker_p,
"switch": 1,
"workers": 1
},
"release_worker": {
"port": 51100,
"port": release_worker_p,
"switch": 1,
"workers": 1
},
"gitlab_merge_request_worker": {
"port": 51200,
"port": gitlab_merge_request_worker_p,
"switch": 0,
"workers": 1
},
"message_insights_worker": {
"port": 51300,
"port": message_insights_worker_p,
"switch": 0,
"workers": 1,
"insight_days": 30,
"models_dir": "message_models"
},
"pull_request_analysis_worker": {
"port": 51400,
"port": pull_request_analysis_worker_p,
"switch": 0,
"workers": 1,
"insight_days": 30
},
"discourse_analysis_worker":{
"port" : 51500,
"port" : discourse_analysis_worker_p,
"switch": 0,
"workers": 1
},
"message_insights_worker": {
"port": 51300,
"port": message_insights_worker_p,
"switch": 0,
"workers": 1,
"insight_days": 30,
"models_dir": "message_models"
},
"pull_request_analysis_worker": {
"port": 51400,
"port": pull_request_analysis_worker_p,
"switch": 0,
"workers": 1,
"insight_days": 30
},
"discourse_analysis_worker":{
"port" : 51500,
"port" : discourse_analysis_worker_p,
"switch": 0,
"workers": 1
},
"clustering_worker": {
"port": 51600,
"port": clustering_worker_p,
"switch": 0,
"workers": 1,
"max_df" : 0.9,
Expand Down Expand Up @@ -335,13 +364,13 @@
"Server": {
"cache_expire": "3600",
"host": "0.0.0.0",
"port": "5000",
"port": main_port,
"workers": 4,
"timeout": 60
},
"Frontend": {
"host": "0.0.0.0",
"port": "5000"
"port": main_port
},
"Logging": {
"logs_directory": "logs/",
Expand Down
45 changes: 37 additions & 8 deletions augur/housekeeper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,17 +324,38 @@ def prep_jobs(self):
def update_url_redirects(self):
if 'switch' in self.update_redirects and self.update_redirects['switch'] == 1 and 'repo_group_id' in self.update_redirects:
repos_urls = self.get_repos_urls(self.update_redirects['repo_group_id'])
if self.update_redirects['repo_group_id'] == 0:
logger.info("Repo Group Set to Zero for URL Updates")
else:
logger.info("Repo Group ID Specified.")
for url in repos_urls:
r = requests.get(url)
check_for_update = url != r.url
if check_for_update:
self.update_repo_url(url, r.url, self.update_redirects['repo_group_id'])
url = self.trim_git_suffix(url)
if url:
r = requests.get(url)
check_for_update = url != r.url
if check_for_update:
self.update_repo_url(url, r.url, self.update_redirects['repo_group_id'])

def trim_git_suffix(self, url):
if url.endswith('.git'):
url = url.replace('.git', '')
elif url.endswith('.github.io'):
url = url.replace('.github.io', '')
elif url.endswith('/.github'):
url = ''
return url

def get_repos_urls(self, repo_group_id):
repos_sql = s.sql.text("""
SELECT repo_git FROM repo
WHERE repo_group_id = ':repo_group_id'
""")
if self.update_redirects['repo_group_id'] == 0:
repos_sql = s.sql.text("""
SELECT repo_git FROM repo
""")
logger.info("repo_group_id is 0")
else:
repos_sql = s.sql.text("""
SELECT repo_git FROM repo
WHERE repo_group_id = ':repo_group_id'
""")

repos = pd.read_sql(repos_sql, self.db, params={'repo_group_id': repo_group_id})

Expand All @@ -344,6 +365,13 @@ def get_repos_urls(self, repo_group_id):
return repos['repo_git']

def update_repo_url(self, old_url, new_url, repo_group_id):
trimmed_new_url = self.trim_git_suffix(new_url)
if not trimmed_new_url:
logger.info("New repo is named .github : {} ... skipping \n".format(new_url))
return
else:
new_url = trimmed_new_url

old_repo_path = Housekeeper.parseRepoName(old_url)
old_repo_group_name = old_repo_path[0]
new_repo_path = Housekeeper.parseRepoName(new_url)
Expand Down Expand Up @@ -378,6 +406,7 @@ def update_repo_url(self, old_url, new_url, repo_group_id):
new_repo_group_id = self.db.execute(insert_sql, new_repo_group_name=new_repo_group_name).fetchone()[0]
logger.info("Inserted repo group {} with id {}\n".format(new_repo_group_name, new_repo_group_id))

new_repo_group_id = '%s' % new_repo_group_id
update_sql = s.sql.text("""
UPDATE repo SET repo_git = :new_url, repo_path = NULL, repo_name = NULL, repo_status = 'New', repo_group_id = :new_repo_group_id
WHERE repo_git = :old_url
Expand Down
Loading

0 comments on commit eef9a54

Please sign in to comment.