From 1e2b0c040d68d9daeb2532a059002198b2098fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Wed, 2 Nov 2022 00:57:01 +0100 Subject: [PATCH] feat: implement collaborators --- README.md | 2 +- lib/args.js | 2 +- lib/github/rest.js | 22 ++++- lib/scraper/collaborators.js | 71 ++++++++++++++++ lib/scraper/extended-summarize.js | 45 ---------- lib/scraper/status.js | 2 +- lib/scraper/summarize.js | 108 ++++++++++++++++++++++++ package-lock.json | 31 ++++++- package.json | 1 + templates/graphql/collaborators.graphql | 8 ++ templates/graphql/summarize.graphql | 27 ++++++ 11 files changed, 266 insertions(+), 53 deletions(-) create mode 100644 lib/scraper/collaborators.js delete mode 100644 lib/scraper/extended-summarize.js create mode 100644 templates/graphql/collaborators.graphql diff --git a/README.md b/README.md index cf35169..650fdaf 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Authentication: Scape settings: -i, --interval scrape interval [number] [default: 600] -s, --spread spread request over interval [boolean] [default: false] - -S, --scraper enable or disable scraper [array] [default: ["summarize","extended-summarize","rate-limit","contributors","status","traffic-clones","traffic-top-paths","traffic-top-referrers","traffic-views"]] + -S, --scraper enable or disable scraper [array] [default: ["collaborators","summarize","rate-limit","contributors","status","traffic-clones","traffic-top-paths","traffic-top-referrers","traffic-views"]] Scape targets: -o, --organization GitHub organization to scrape. Can be defined multiple times or comma separated list [array] [default: []] diff --git a/lib/args.js b/lib/args.js index 1c51882..8ea3484 100644 --- a/lib/args.js +++ b/lib/args.js @@ -48,8 +48,8 @@ module.exports.argv = require('yargs') group: 'Scape settings:', array: true, default: [ + 'collaborators', 'summarize', - 'extended-summarize', 'rate-limit', 'contributors', 'status', diff --git a/lib/github/rest.js b/lib/github/rest.js index 6b562f2..899bde7 100644 --- a/lib/github/rest.js +++ b/lib/github/rest.js @@ -1,4 +1,5 @@ const { Octokit } = require('@octokit/rest') +const { throttling } = require('@octokit/plugin-throttling') const { version } = require('../../package.json') const { argv } = require('../args') @@ -12,6 +13,25 @@ const options = { info: (message) => logger.verbose(message), warn: (message) => logger.warn(message), error: (message) => logger.error(message) + }, + throttle: { + onRateLimit: (retryAfter, options, octokit, retryCount) => { + octokit.log.warn( + `Request quota exhausted for request ${options.method} ${options.url}` + ) + + if (retryCount < 1) { + // only retries once + octokit.log.info(`Retrying after ${retryAfter} seconds!`) + return true + } + }, + onSecondaryRateLimit: (retryAfter, options, octokit) => { + // does not retry, only logs a warning + octokit.log.warn( + `SecondaryRateLimit detected for request ${options.method} ${options.url}` + ) + } } } @@ -36,6 +56,6 @@ switch (argv.authStrategy) { break } -const octokit = new Octokit(options) +const octokit = new (Octokit.plugin(throttling))(options) module.exports = octokit diff --git a/lib/scraper/collaborators.js b/lib/scraper/collaborators.js new file mode 100644 index 0000000..3699e12 --- /dev/null +++ b/lib/scraper/collaborators.js @@ -0,0 +1,71 @@ +const Prometheus = require('prom-client') +const fs = require('fs') +const graphql = require('../github/graphql') +const logger = require('../logger') +const helpers = require('../helpers') + +const maxRepositoriesPerScrape = 100 + +class Status { + constructor () { + this.registerMetrics() + this.loadQueryTemplate() + } + + registerMetrics () { + this.metrics = { + githubRepoCollaboratorGauge: new Prometheus.Gauge({ + name: 'github_repo_collaborator_total', + help: 'total amount of collaborators for given repository', + labelNames: ['owner', 'repository', 'affiliation'] + }) + } + } + + loadQueryTemplate () { + this.graphqlQuery = fs.readFileSync('./templates/graphql/collaborators.graphql', 'utf8') + } + + scrapeRepositories (repositories) { + const repositoryChunks = helpers.splitArray(repositories, maxRepositoriesPerScrape) + + repositoryChunks.forEach((repositoryChunk) => { + graphql(this.getQuery(repositoryChunk)) + .then((response) => { + repositoryChunk.forEach((repository) => { + const repoLabel = `repo_${helpers.transformRepositoryNameToGraphQlLabel(repository)}` + + if (!(repoLabel in response)) { + logger.error(`Can't find metrics for repository ${repository} in response.`) + } + + this.process(repository, response[repoLabel]) + }) + }) + .catch((err) => { + logger.error( + `Failed to get collaborators from repository ${repositories.join(', ')}: `, + err + ) + }) + }) + } + + process (repository, repositoryMetrics) { + const [owner] = repository.split('/') + + this.metrics.githubRepoCollaboratorGauge.set({ owner, repository, affiliation: 'direct' }, repositoryMetrics.collaboratorsDirect.totalCount) + this.metrics.githubRepoCollaboratorGauge.set({ owner, repository, affiliation: 'outside' }, repositoryMetrics.collaboratorsOutside.totalCount) + } + + getQuery (repositories) { + return [ + this.graphqlQuery, + '{', + helpers.generateGraphqlRepositoryQueries(repositories), + '}' + ].join('\n') + } +} + +module.exports = Status diff --git a/lib/scraper/extended-summarize.js b/lib/scraper/extended-summarize.js deleted file mode 100644 index 19718e9..0000000 --- a/lib/scraper/extended-summarize.js +++ /dev/null @@ -1,45 +0,0 @@ -const Prometheus = require('prom-client') -const ghRestApi = require('../github/rest') -const logger = require('../logger') - -class ExtendedSummarize { - constructor () { - this.registerMetrics() - } - - registerMetrics () { - this.metrics = { - githubRepoSizeGauge: new Prometheus.Gauge({ - name: 'github_repo_size_kb', - help: 'size for given repository', - labelNames: ['owner', 'repository'] - }), - githubRepoNetworkGauge: new Prometheus.Gauge({ - name: 'github_repo_network_total', - help: 'network size for given repository', - labelNames: ['owner', 'repository'] - }) - } - } - - scrapeRepositories (repositories) { - repositories.forEach((repository) => { - const [owner, repo] = repository.split('/') - - ghRestApi.repos - .get({ owner, repo }) - .then((response) => { - this.metrics.githubRepoSizeGauge.set({ owner, repository }, response.data.size) - this.metrics.githubRepoNetworkGauge.set( - { owner, repository }, - response.data.network_count - ) - }) - .catch((err) => - logger.error(`Failed to scrape repository ${repository} via REST: ${err.message}`, err) - ) - }) - } -} - -module.exports = ExtendedSummarize diff --git a/lib/scraper/status.js b/lib/scraper/status.js index cad5580..3e196d5 100644 --- a/lib/scraper/status.js +++ b/lib/scraper/status.js @@ -47,7 +47,7 @@ class Status { }) .catch((err) => { logger.error( - `Failed to scrape vulnerabilities from repository ${repositories.join(', ')}: `, + `Failed get status from repository ${repositories.join(', ')}: `, err ) }) diff --git a/lib/scraper/summarize.js b/lib/scraper/summarize.js index 67269fb..36af78b 100644 --- a/lib/scraper/summarize.js +++ b/lib/scraper/summarize.js @@ -129,6 +129,56 @@ class Summarize { help: 'Total number of releases for given repository', labelNames: ['owner', 'repository'] }), + githubRepoDiskUsageGauge: new Prometheus.Gauge({ + name: 'github_repo_disk_usage', + help: 'The number of kilobytes this repository occupies on disk.', + labelNames: ['owner', 'repository'] + }), + githubRepoDiscussionsGauge: new Prometheus.Gauge({ + name: 'github_repo_discussions_total', + help: 'Total number of discussions for given repository', + labelNames: ['owner', 'repository'] + }), + githubRepoDeploymentsGauge: new Prometheus.Gauge({ + name: 'github_repo_deployments_total', + help: 'Total number of deployments for given repository', + labelNames: ['owner', 'repository'] + }), + githubRepoEnvironmentsGauge: new Prometheus.Gauge({ + name: 'github_repo_environments_total', + help: 'Total number of deployments for given repository', + labelNames: ['owner', 'repository'] + }), + githubRepoMentionableUsersGauge: new Prometheus.Gauge({ + name: 'github_repo_mentionable_users_total', + help: 'Total number of mentionable users for given repository', + labelNames: ['owner', 'repository'] + }), + githubRepoCollaboratorsGauge: new Prometheus.Gauge({ + name: 'github_repo_collaborators_total', + help: 'Total number of collaborators for given repository', + labelNames: ['owner', 'repository'] + }), + githubRepoMilestonesTotalGauge: new Prometheus.Gauge({ + name: 'github_repo_milestones_total', + help: 'Total number of collaborators for given repository', + labelNames: ['owner', 'repository'] + }), + githubRepoMilestonesProgressPercentageGauge: new Prometheus.Gauge({ + name: 'github_repo_milestone_percent', + help: 'Percent of milestone inside the given repository', + labelNames: ['owner', 'repository', 'number', 'title'] + }), + githubRepoMilestonesStateGauge: new Prometheus.Gauge({ + name: 'github_repo_milestone_state', + help: 'State of milestone inside the given repository', + labelNames: ['owner', 'repository', 'number', 'title'] + }), + githubRepoMilestonesIssuesTotalGauge: new Prometheus.Gauge({ + name: 'github_repo_milestone_issues_total', + help: 'Total issue count of milestone inside the given repository', + labelNames: ['owner', 'repository', 'number', 'title'] + }), // TODO: label for dismissed alerts githubRepoVulnerabilitiesGauge: new Prometheus.Gauge({ @@ -300,6 +350,34 @@ class Summarize { { owner, repository }, repositoryMetrics.releases.totalCount ) + this.metrics.githubRepoDiskUsageGauge.set( + { owner, repository }, + repositoryMetrics.diskUsage + ) + this.metrics.githubRepoDiscussionsGauge.set( + { owner, repository }, + repositoryMetrics.discussions.totalCount + ) + this.metrics.githubRepoDeploymentsGauge.set( + { owner, repository }, + repositoryMetrics.deployments.totalCount + ) + this.metrics.githubRepoEnvironmentsGauge.set( + { owner, repository }, + repositoryMetrics.environments.totalCount + ) + this.metrics.githubRepoMentionableUsersGauge.set( + { owner, repository }, + repositoryMetrics.mentionableUsers.totalCount + ) + this.metrics.githubRepoCollaboratorsGauge.set( + { owner, repository }, + repositoryMetrics.collaborators.totalCount + ) + this.metrics.githubRepoMilestonesTotalGauge.set( + { owner, repository }, + repositoryMetrics.milestones.totalCount + ) repositoryMetrics.releases.nodes.forEach((release) => { release.releaseAssets.nodes.forEach((asset) => { @@ -328,6 +406,36 @@ class Summarize { ) }) + repositoryMetrics.milestones.nodes.forEach((milestone) => { + this.metrics.githubRepoMilestonesStateGauge.set( + { + owner, + repository, + title: milestone.title, + number: milestone.number + }, + milestone.state === 'OPEN' ? 0 : 1 + ) + this.metrics.githubRepoMilestonesProgressPercentageGauge.set( + { + owner, + repository, + title: milestone.title, + number: milestone.number + }, + milestone.progressPercentage + ) + this.metrics.githubRepoMilestonesIssuesTotalGauge.set( + { + owner, + repository, + title: milestone.title, + number: milestone.number + }, + milestone.issues.totalCount + ) + }) + this.metrics.githubRepoScrapedGauge.set({ owner, repository }, 1) } catch (err) { this.metrics.githubRepoScrapedGauge.set({ owner, repository }, 0) diff --git a/package-lock.json b/package-lock.json index 407b2df..ed28c78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@octokit/auth-oauth-app": "^5.0.4", "@octokit/auth-token": "^3.0.2", "@octokit/graphql": "^5.0.4", + "@octokit/plugin-throttling": "^4.3.2", "@octokit/rest": "^19.0.5", "dotenv": "^16.0.3", "prom-client": "^14.1.0", @@ -1202,6 +1203,21 @@ "@octokit/core": ">=3" } }, + "node_modules/@octokit/plugin-throttling": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-4.3.2.tgz", + "integrity": "sha512-ZaCK599h3tzcoy0Jtdab95jgmD7X9iAk59E2E7hYKCAmnURaI4WpzwL9vckImilybUGrjY1JOWJapDs2N2D3vw==", + "dependencies": { + "@octokit/types": "^8.0.0", + "bottleneck": "^2.15.3" + }, + "engines": { + "node": ">= 14" + }, + "peerDependencies": { + "@octokit/core": "^4.0.0" + } + }, "node_modules/@octokit/request": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-6.2.2.tgz", @@ -1853,8 +1869,7 @@ "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", - "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", - "dev": true + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, "node_modules/brace-expansion": { "version": "1.1.11", @@ -12814,6 +12829,15 @@ "deprecation": "^2.3.1" } }, + "@octokit/plugin-throttling": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-4.3.2.tgz", + "integrity": "sha512-ZaCK599h3tzcoy0Jtdab95jgmD7X9iAk59E2E7hYKCAmnURaI4WpzwL9vckImilybUGrjY1JOWJapDs2N2D3vw==", + "requires": { + "@octokit/types": "^8.0.0", + "bottleneck": "^2.15.3" + } + }, "@octokit/request": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-6.2.2.tgz", @@ -13321,8 +13345,7 @@ "bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", - "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", - "dev": true + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, "brace-expansion": { "version": "1.1.11", diff --git a/package.json b/package.json index 18cd72a..ab642ae 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@octokit/auth-oauth-app": "^5.0.4", "@octokit/auth-token": "^3.0.2", "@octokit/graphql": "^5.0.4", + "@octokit/plugin-throttling": "^4.3.2", "@octokit/rest": "^19.0.5", "dotenv": "^16.0.3", "prom-client": "^14.1.0", diff --git a/templates/graphql/collaborators.graphql b/templates/graphql/collaborators.graphql new file mode 100644 index 0000000..4e8d58f --- /dev/null +++ b/templates/graphql/collaborators.graphql @@ -0,0 +1,8 @@ +fragment repositoryFragment on Repository { + collaboratorsDirect: collaborators(first: 0, affiliation: DIRECT) { + totalCount + } + collaboratorsOutside: collaborators(first: 0, affiliation: OUTSIDE) { + totalCount + } +} diff --git a/templates/graphql/summarize.graphql b/templates/graphql/summarize.graphql index 26aea7a..8d71081 100644 --- a/templates/graphql/summarize.graphql +++ b/templates/graphql/summarize.graphql @@ -83,4 +83,31 @@ fragment repositoryFragment on Repository { vulnerabilityAlerts(first: 0) { totalCount } + discussions(first: 0) { + totalCount + } + deployments(first: 0) { + totalCount + } + environments(first: 0) { + totalCount + } + mentionableUsers(first: 0) { + totalCount + } + collaborators(first: 0) { + totalCount + } + milestones(first: 100) { + totalCount + nodes { + number + title + progressPercentage + state + issues(first: 0) { + totalCount + } + } + } }