feat: implement collaborators

README.md

@@ -57,7 +57,7 @@ Authentication:
 Scape settings:
   -i, --interval  scrape interval  [number] [default: 600]
   -s, --spread    spread request over interval  [boolean] [default: false]
-  -S, --scraper   enable or disable scraper  [array] [default: ["summarize","extended-summarize","rate-limit","contributors","status","traffic-clones","traffic-top-paths","traffic-top-referrers","traffic-views"]]
+  -S, --scraper   enable or disable scraper  [array] [default: ["collaborators","summarize","rate-limit","contributors","status","traffic-clones","traffic-top-paths","traffic-top-referrers","traffic-views"]]
 
 Scape targets:
   -o, --organization  GitHub organization to scrape. Can be defined multiple times or comma separated list  [array] [default: []]

lib/args.js

@@ -48,8 +48,8 @@ module.exports.argv = require('yargs')
     group: 'Scape settings:',
     array: true,
     default: [
+      'collaborators',
       'summarize',
-      'extended-summarize',
       'rate-limit',
       'contributors',
       'status',

lib/github/rest.js

@@ -1,4 +1,5 @@
 const { Octokit } = require('@octokit/rest')
+const { throttling } = require('@octokit/plugin-throttling')
 
 const { version } = require('../../package.json')
 const { argv } = require('../args')
@@ -12,6 +13,25 @@ const options = {
     info: (message) => logger.verbose(message),
     warn: (message) => logger.warn(message),
     error: (message) => logger.error(message)
+  },
+  throttle: {
+    onRateLimit: (retryAfter, options, octokit, retryCount) => {
+      octokit.log.warn(
+        `Request quota exhausted for request ${options.method} ${options.url}`
+      )
+
+      if (retryCount < 1) {
+        // only retries once
+        octokit.log.info(`Retrying after ${retryAfter} seconds!`)
+        return true
+      }
+    },
+    onSecondaryRateLimit: (retryAfter, options, octokit) => {
+      // does not retry, only logs a warning
+      octokit.log.warn(
+        `SecondaryRateLimit detected for request ${options.method} ${options.url}`
+      )
+    }
   }
 }
 
@@ -36,6 +56,6 @@ switch (argv.authStrategy) {
     break
 }
 
-const octokit = new Octokit(options)
+const octokit = new (Octokit.plugin(throttling))(options)
 
 module.exports = octokit

lib/scraper/collaborators.js

@@ -0,0 +1,71 @@
+const Prometheus = require('prom-client')
+const fs = require('fs')
+const graphql = require('../github/graphql')
+const logger = require('../logger')
+const helpers = require('../helpers')
+
+const maxRepositoriesPerScrape = 100
+
+class Status {
+  constructor () {
+    this.registerMetrics()
+    this.loadQueryTemplate()
+  }
+
+  registerMetrics () {
+    this.metrics = {
+      githubRepoCollaboratorGauge: new Prometheus.Gauge({
+        name: 'github_repo_collaborator_total',
+        help: 'total amount of collaborators for given repository',
+        labelNames: ['owner', 'repository', 'affiliation']
+      })
+    }
+  }
+
+  loadQueryTemplate () {
+    this.graphqlQuery = fs.readFileSync('./templates/graphql/collaborators.graphql', 'utf8')
+  }
+
+  scrapeRepositories (repositories) {
+    const repositoryChunks = helpers.splitArray(repositories, maxRepositoriesPerScrape)
+
+    repositoryChunks.forEach((repositoryChunk) => {
+      graphql(this.getQuery(repositoryChunk))
+        .then((response) => {
+          repositoryChunk.forEach((repository) => {
+            const repoLabel = `repo_${helpers.transformRepositoryNameToGraphQlLabel(repository)}`
+
+            if (!(repoLabel in response)) {
+              logger.error(`Can't find metrics for repository ${repository} in response.`)
+            }
+
+            this.process(repository, response[repoLabel])
+          })
+        })
+        .catch((err) => {
+          logger.error(
+            `Failed to get collaborators from repository ${repositories.join(', ')}: `,
+            err
+          )
+        })
+    })
+  }
+
+  process (repository, repositoryMetrics) {
+    const [owner] = repository.split('/')
+
+    this.metrics.githubRepoCollaboratorGauge.set({ owner, repository, affiliation: 'direct' }, repositoryMetrics.collaboratorsDirect.totalCount)
+    this.metrics.githubRepoCollaboratorGauge.set({ owner, repository, affiliation: 'outside' }, repositoryMetrics.collaboratorsOutside.totalCount)
+  }
+
+  getQuery (repositories) {
+    return [
+      this.graphqlQuery,
+      '{',
+      helpers.generateGraphqlRepositoryQueries(repositories),
+      '}'
+    ].join('\n')
+  }
+}
+
+module.exports = Status

lib/scraper/status.js

@@ -47,7 +47,7 @@ class Status {
         })
         .catch((err) => {
           logger.error(
-            `Failed to scrape vulnerabilities from repository ${repositories.join(', ')}: `,
+            `Failed get status from repository ${repositories.join(', ')}: `,
             err
           )
         })

lib/scraper/summarize.js

@@ -129,6 +129,56 @@ class Summarize {
         help: 'Total number of releases for given repository',
         labelNames: ['owner', 'repository']
       }),
+      githubRepoDiskUsageGauge: new Prometheus.Gauge({
+        name: 'github_repo_disk_usage',
+        help: 'The number of kilobytes this repository occupies on disk.',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoDiscussionsGauge: new Prometheus.Gauge({
+        name: 'github_repo_discussions_total',
+        help: 'Total number of discussions for given repository',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoDeploymentsGauge: new Prometheus.Gauge({
+        name: 'github_repo_deployments_total',
+        help: 'Total number of deployments for given repository',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoEnvironmentsGauge: new Prometheus.Gauge({
+        name: 'github_repo_environments_total',
+        help: 'Total number of deployments for given repository',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoMentionableUsersGauge: new Prometheus.Gauge({
+        name: 'github_repo_mentionable_users_total',
+        help: 'Total number of mentionable users for given repository',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoCollaboratorsGauge: new Prometheus.Gauge({
+        name: 'github_repo_collaborators_total',
+        help: 'Total number of collaborators for given repository',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoMilestonesTotalGauge: new Prometheus.Gauge({
+        name: 'github_repo_milestones_total',
+        help: 'Total number of collaborators for given repository',
+        labelNames: ['owner', 'repository']
+      }),
+      githubRepoMilestonesProgressPercentageGauge: new Prometheus.Gauge({
+        name: 'github_repo_milestone_percent',
+        help: 'Percent of milestone inside the given repository',
+        labelNames: ['owner', 'repository', 'number', 'title']
+      }),
+      githubRepoMilestonesStateGauge: new Prometheus.Gauge({
+        name: 'github_repo_milestone_state',
+        help: 'State of milestone inside the given repository',
+        labelNames: ['owner', 'repository', 'number', 'title']
+      }),
+      githubRepoMilestonesIssuesTotalGauge: new Prometheus.Gauge({
+        name: 'github_repo_milestone_issues_total',
+        help: 'Total issue count of milestone inside the given repository',
+        labelNames: ['owner', 'repository', 'number', 'title']
+      }),
 
       // TODO: label for dismissed alerts
       githubRepoVulnerabilitiesGauge: new Prometheus.Gauge({
@@ -300,6 +350,34 @@ class Summarize {
         { owner, repository },
         repositoryMetrics.releases.totalCount
       )
+      this.metrics.githubRepoDiskUsageGauge.set(
+        { owner, repository },
+        repositoryMetrics.diskUsage
+      )
+      this.metrics.githubRepoDiscussionsGauge.set(
+        { owner, repository },
+        repositoryMetrics.discussions.totalCount
+      )
+      this.metrics.githubRepoDeploymentsGauge.set(
+        { owner, repository },
+        repositoryMetrics.deployments.totalCount
+      )
+      this.metrics.githubRepoEnvironmentsGauge.set(
+        { owner, repository },
+        repositoryMetrics.environments.totalCount
+      )
+      this.metrics.githubRepoMentionableUsersGauge.set(
+        { owner, repository },
+        repositoryMetrics.mentionableUsers.totalCount
+      )
+      this.metrics.githubRepoCollaboratorsGauge.set(
+        { owner, repository },
+        repositoryMetrics.collaborators.totalCount
+      )
+      this.metrics.githubRepoMilestonesTotalGauge.set(
+        { owner, repository },
+        repositoryMetrics.milestones.totalCount
+      )
 
       repositoryMetrics.releases.nodes.forEach((release) => {
         release.releaseAssets.nodes.forEach((asset) => {
@@ -328,6 +406,36 @@ class Summarize {
         )
       })
 
+      repositoryMetrics.milestones.nodes.forEach((milestone) => {
+        this.metrics.githubRepoMilestonesStateGauge.set(
+          {
+            owner,
+            repository,
+            title: milestone.title,
+            number: milestone.number
+          },
+          milestone.state === 'OPEN' ? 0 : 1
+        )
+        this.metrics.githubRepoMilestonesProgressPercentageGauge.set(
+          {
+            owner,
+            repository,
+            title: milestone.title,
+            number: milestone.number
+          },
+          milestone.progressPercentage
+        )
+        this.metrics.githubRepoMilestonesIssuesTotalGauge.set(
+          {
+            owner,
+            repository,
+            title: milestone.title,
+            number: milestone.number
+          },
+          milestone.issues.totalCount
+        )
+      })
+
       this.metrics.githubRepoScrapedGauge.set({ owner, repository }, 1)
     } catch (err) {
       this.metrics.githubRepoScrapedGauge.set({ owner, repository }, 0)

package.json

@@ -55,6 +55,7 @@
     "@octokit/auth-oauth-app": "^5.0.4",
     "@octokit/auth-token": "^3.0.2",
     "@octokit/graphql": "^5.0.4",
+    "@octokit/plugin-throttling": "^4.3.2",
     "@octokit/rest": "^19.0.5",
     "dotenv": "^16.0.3",
     "prom-client": "^14.1.0",

templates/graphql/collaborators.graphql

@@ -0,0 +1,8 @@
+fragment repositoryFragment on Repository {
+  collaboratorsDirect: collaborators(first: 0, affiliation: DIRECT) {
+    totalCount
+  }
+  collaboratorsOutside: collaborators(first: 0, affiliation: OUTSIDE) {
+    totalCount
+  }
+}