Skip to content

Commit

Permalink
Update GitStats (#47)
Browse files Browse the repository at this point in the history
  • Loading branch information
@R055A
R055A committed Oct 20, 2023
1 parent 37b2942 commit cfed0e0
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 83 deletions.
33 changes: 33 additions & 0 deletions .github/workflows/pull_request_code_convention.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: Pull Request Lint Check

on:
pull_request:
branches:
- master

permissions:
contents: read

jobs:

ruff:

runs-on: ubuntu-latest

steps:

# Run using Python 3.11 for consistency
- name: Set up Python 3.11
uses: actions/setup-python@v3
with:
python-version: '3.11'
architecture: 'x64'
cache: pip

# Install Ruff
- name: Install ruff
run: pip install --user ruff

# Use Ruff to lint all files
- name: Ruff lint
run: ruff --output-format=github .
15 changes: 8 additions & 7 deletions src/github_api_queries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import Dict, Optional, List
from json import loads


###############################################################################
# GitHubApiQueries class
###############################################################################
Expand Down Expand Up @@ -56,7 +57,7 @@ async def query(self, generated_query: str) -> Dict:

if result is not None:
return result
except:
except ConnectionError:
print("aiohttp failed for GraphQL query")

# Fall back on non-async requests
Expand Down Expand Up @@ -102,7 +103,7 @@ async def query_rest(self, path: str, params: Optional[Dict] = None) -> Dict:

if result is not None:
return result
except:
except ConnectionError:
print("aiohttp failed for REST query attempt #" + str(i + 1))

# Fall back on non-async requests
Expand Down Expand Up @@ -142,9 +143,9 @@ def repos_overview(
direction: DESC
}},
after: {
"null" if owned_cursor is None
else '"' + owned_cursor + '"'
}) {{
"null" if owned_cursor is None
else '"' + owned_cursor + '"'
}) {{
pageInfo {{
hasNextPage
endCursor
Expand Down Expand Up @@ -187,8 +188,8 @@ def repos_overview(
PULL_REQUEST_REVIEW
]
after: {
"null" if contrib_cursor is None
else '"' + contrib_cursor + '"'}) {{
"null" if contrib_cursor is None
else '"' + contrib_cursor + '"'}) {{
pageInfo {{
hasNextPage
endCursor
Expand Down
169 changes: 93 additions & 76 deletions src/github_repo_stats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ class GitHubRepoStats(object):
Retrieve and store statistics about GitHub usage.
"""

__DATE_FORMAT = "%Y-%m-%d"
__EXCLUDED_USER_NAMES = [
_DATE_FORMAT = "%Y-%m-%d"
_EXCLUDED_USER_NAMES = [
"dependabot[bot]"
] # exclude bot data from being included in statistical calculations
_NO_NAME = "No Name"

def __init__(self, environment_vars: EnvironmentVariables, session: ClientSession):
self.environment_vars: EnvironmentVariables = environment_vars
Expand Down Expand Up @@ -146,7 +147,7 @@ async def get_stats(self) -> None:
self._name = (
raw_results.get("data", {})
.get("viewer", {})
.get("login", "No Name")
.get("login", self._NO_NAME)
)

contrib_repos = (
Expand All @@ -163,39 +164,7 @@ async def get_stats(self) -> None:
if not self.environment_vars.exclude_contrib_repos:
repos += contrib_repos.get("nodes", [])

for repo in repos:
if not repo or await self.is_repo_type_excluded(repo):
continue

name = repo.get("nameWithOwner")
if await self.is_repo_name_invalid(name):
continue
self._repos.add(name)

self._stargazers += repo.get("stargazers").get("totalCount", 0)
self._forks += repo.get("forkCount", 0)

if repo.get("isEmpty"):
self._empty_repos.add(name)
continue

for lang in repo.get("languages", {}).get("edges", []):
name = lang.get("node", {}).get("name", "Other")
languages = await self.languages

if name in self.environment_vars.exclude_langs:
self._excluded_languages.add(name)
continue

if name in languages:
languages[name]["size"] += lang.get("size", 0)
languages[name]["occurrences"] += 1
else:
languages[name] = {
"size": lang.get("size", 0),
"occurrences": 1,
"color": lang.get("node", {}).get("color"),
}
await self.repo_stats(repos)

is_cur_owned = owned_repos.get("pageInfo", {}).get("hasNextPage", False)
is_cur_contrib = contrib_repos.get("pageInfo", {}).get("hasNextPage", False)
Expand All @@ -210,49 +179,91 @@ async def get_stats(self) -> None:
else:
break

if not self.environment_vars.exclude_contrib_repos:
env_repos = self.environment_vars.manually_added_repos
lang_cols = self.queries.get_language_colors()
await self.manually_added_repo_stats()

for repo in env_repos:
if await self.is_repo_name_invalid(repo):
continue
self._repos.add(repo)
# TODO: Improve languages to scale by number of contributions to specific filetypes
langs_total = sum([v.get("size", 0) for v in self._languages.values()])
for k, v in self._languages.items():
v["prop"] = 100 * (v.get("size", 0) / langs_total)

repo_stats = await self.queries.query_rest(f"/repos/{repo}")
if await self.is_repo_type_excluded(repo_stats):
continue
async def repo_stats(self, repos) -> None:
"""
Gathers statistical data from fetches for repos user is associated with on GitHub
"""
for repo in repos:
if not repo or await self.is_repo_type_excluded(repo):
continue

name = repo.get("nameWithOwner")
if await self.is_repo_name_invalid(name):
continue
self._repos.add(name)

self._stargazers += repo_stats.get("stargazers_count", 0)
self._forks += repo_stats.get("forks", 0)
self._stargazers += repo.get("stargazers").get("totalCount", 0)
self._forks += repo.get("forkCount", 0)

if repo_stats.get("size") == 0:
self._empty_repos.add(repo)
if repo.get("isEmpty"):
self._empty_repos.add(name)
continue

for lang in repo.get("languages", {}).get("edges", []):
name = lang.get("node", {}).get("name", "Other")
languages = await self.languages

if name in self.environment_vars.exclude_langs:
self._excluded_languages.add(name)
continue

# TODO: Improve languages to scale by number of contributions to specific filetypes
if repo_stats.get("language"):
langs = await self.queries.query_rest(f"/repos/{repo}/languages")
if name in languages:
languages[name]["size"] += lang.get("size", 0)
languages[name]["occurrences"] += 1
else:
languages[name] = {
"size": lang.get("size", 0),
"occurrences": 1,
"color": lang.get("node", {}).get("color"),
}

for lang, size in langs.items():
languages = await self.languages
async def manually_added_repo_stats(self) -> None:
"""
Gathers statistical data from fetches for manually added repos otherwise not fetched by user association
"""
lang_cols = self.queries.get_language_colors()

if lang in self.environment_vars.exclude_langs:
continue
for repo in self.environment_vars.manually_added_repos:
if await self.is_repo_name_invalid(repo):
continue
self._repos.add(repo)

if lang in languages:
languages[lang]["size"] += size
languages[lang]["occurrences"] += 1
else:
languages[lang] = {
"size": size,
"occurrences": 1,
"color": lang_cols.get(lang).get("color"),
}
repo_stats = await self.queries.query_rest(f"/repos/{repo}")
if await self.is_repo_type_excluded(repo_stats):
continue

langs_total = sum([v.get("size", 0) for v in self._languages.values()])
for k, v in self._languages.items():
v["prop"] = 100 * (v.get("size", 0) / langs_total)
self._stargazers += repo_stats.get("stargazers_count", 0)
self._forks += repo_stats.get("forks", 0)

if repo_stats.get("size") == 0:
self._empty_repos.add(repo)
continue

if repo_stats.get("language"):
langs = await self.queries.query_rest(f"/repos/{repo}/languages")

for lang, size in langs.items():
languages = await self.languages

if lang in self.environment_vars.exclude_langs:
continue

if lang in languages:
languages[lang]["size"] += size
languages[lang]["occurrences"] += 1
else:
languages[lang] = {
"size": size,
"occurrences": 1,
"color": lang_cols.get(lang).get("color"),
}

@property
async def name(self) -> str:
Expand Down Expand Up @@ -388,7 +399,7 @@ async def lines_changed(self) -> Tuple[int, int]:
"""
if self._users_lines_changed is not None:
return self._users_lines_changed
_, ghosted_collab_repos = await self.raw_collaborators()
_, collab_repos = await self.raw_collaborators()
slave_status_repos = self.environment_vars.more_collab_repos

contributor_set = set()
Expand Down Expand Up @@ -417,7 +428,7 @@ async def lines_changed(self) -> Tuple[int, int]:

if (
author != self.environment_vars.username
and author not in self.__EXCLUDED_USER_NAMES
and author not in self._EXCLUDED_USER_NAMES
):
for week in author_obj.get("weeks", []):
other_authors_total_changes += week.get("a", 0)
Expand All @@ -429,13 +440,15 @@ async def lines_changed(self) -> Tuple[int, int]:
author_total_additions += author_additions
author_total_deletions += author_deletions

# calculate average author's contributions to each repository with more than one contributor (or should be)
# calculate average author's contributions to each repository with at least one other collaborator
if (
repo not in self.environment_vars.exclude_collab_repos
and (author_additions + author_deletions) > 0
and (
other_authors_total_changes > 0
or repo in ghosted_collab_repos | slave_status_repos
or repo
in collab_repos
| slave_status_repos # either collaborators are ghosting or no show in repo
)
):
repo_total_changes = (
Expand Down Expand Up @@ -477,8 +490,8 @@ async def views(self) -> int:
return self._views

last_viewed = self.environment_vars.repo_last_viewed
today = date.today().strftime(self.__DATE_FORMAT)
yesterday = (date.today() - timedelta(1)).strftime(self.__DATE_FORMAT)
today = date.today().strftime(self._DATE_FORMAT)
yesterday = (date.today() - timedelta(1)).strftime(self._DATE_FORMAT)
dates = {last_viewed, yesterday}

today_view_count = 0
Expand Down Expand Up @@ -530,11 +543,15 @@ async def raw_collaborators(self) -> (Set, Set):

for repo in await self.repos:
r = await self.queries.query_rest(f"/repos/{repo}/collaborators")
collab_count = 0

for obj in r:
if isinstance(obj, dict):
collab_count += 1
self._collaborator_set.add(obj.get("login"))
self._collab_repos.add(repo)

if collab_count > 1:
self._collab_repos.add(repo)

return self._collaborator_set, self._collab_repos

Expand Down

0 comments on commit cfed0e0

Please sign in to comment.