Skip to content

Commit

Permalink
Merge pull request #34 from secure-dashboards/feat/add-seeds
Browse files Browse the repository at this point in the history
  • Loading branch information
@UlisesGascon
UlisesGascon authored Dec 6, 2024
2 parents 8c2ad16 + 42bfcba commit 0e91fd0
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 85 deletions.
11 changes: 0 additions & 11 deletions __tests__/cli/workflows.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,6 @@ describe('run GENERIC - Non-Interactive Mode', () => {
})

describe('run update-github-orgs', () => {
// // Mock inquirer for testing
// jest.spyOn(inquirer, 'prompt').mockImplementation(async (questions) => {
// const questionMap = {
// 'What is the name of the workflow?': 'update-github-orgs'
// }
// return questions.reduce((acc, question) => {
// acc[question.name] = questionMap[question.message]
// return acc
// }, {})
// })

test('Should throw an error when no Github orgs are stored in the database', async () => {
const projects = await getAllProjects(knex)
expect(projects.length).toBe(0)
Expand Down
Empty file removed src/database/seeds/.placeholder
View file
Empty file.
28 changes: 28 additions & 0 deletions src/database/seeds/index.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
const {
sampleGithubOrg,
sampleGithubRepository
} = require('../../../__fixtures__')
const {
resetDatabase,
addProject,
addGithubOrg,
addGithubRepo
} = require('../../../__utils__')
const { github } = require('../../providers')

exports.seed = async (knex) => {
// Clean up the database
await resetDatabase(knex)

// Add a project
const project = await addProject(knex, {
name: 'github',
category: 'impact'
})

// Add a GitHub organization
const githubOrg = await addGithubOrg(knex, { ...github.mappers.org(sampleGithubOrg), project_id: project.id })

// Add GitHub repository
await addGithubRepo(knex, { ...github.mappers.repo(sampleGithubRepository), github_organization_id: githubOrg.id })
}
73 changes: 72 additions & 1 deletion src/providers/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const { Octokit } = require('octokit')
const { ensureGithubToken } = require('../utils')
const { simplifyObject } = require('@ulisesgascon/simplify-object')
const debug = require('debug')('providers:github')

const fetchOrgByLogin = async (login) => {
Expand Down Expand Up @@ -56,7 +57,77 @@ const fetchRepoByFullName = async (fullName) => {
const github = {
fetchOrgByLogin,
fetchOrgReposListByLogin,
fetchRepoByFullName
fetchRepoByFullName,
mappers: {
org: (data) => {
const mappedData = simplifyObject(data, {
// Relevant fields for organizations
include: [
'login', 'node_id', 'url', 'avatar_url',
'description', 'name', 'company', 'blog',
'location', 'twitter_username', 'is_verified',
'has_organization_projects', 'has_repository_projects',
'public_repos', 'public_gists', 'followers',
'following', 'html_url', 'total_private_repos',
'owned_private_repos', 'private_gists', 'disk_usage',
'collaborators', 'default_repository_permission',
'default_repository_permission', 'members_can_create_repositories',
'two_factor_requirement_enabled', 'members_allowed_repository_creation_type',
'members_can_create_public_repositories', 'members_can_create_private_repositories',
'members_can_create_internal_repositories', 'members_can_create_pages',
'members_can_create_public_pages', 'members_can_create_private_pages',
'members_can_fork_private_repositories', 'web_commit_signoff_required',
'deploy_keys_enabled_for_repositories', 'dependency_graph_enabled_for_new_repositories',
'dependabot_alerts_enabled_for_new_repositories', 'dependabot_security_updates_enabled_for_new_repositories',
'advanced_security_enabled_for_new_repositories', 'secret_scanning_enabled_for_new_repositories',
'secret_scanning_push_protection_enabled_for_new_repositories', 'secret_scanning_push_protection_custom_link',
'secret_scanning_push_protection_custom_link_enabled']
})
// Additional fields that have different names in the database
mappedData.github_org_id = data.id
mappedData.github_created_at = data.created_at
mappedData.github_updated_at = data.updated_at
mappedData.github_archived_at = data.archived_at
return mappedData
},
repo: (data) => {
const mappedData = simplifyObject(data, {
include: [
'node_id', 'name', 'full_name',
'html_url', 'description', 'fork', 'url',
'git_url', 'ssh_url', 'clone_url', 'svn_url',
'homepage', 'size', 'stargazers_count', 'watchers_count',
'language', 'has_issues', 'has_projects', 'has_downloads',
'has_wiki', 'has_pages', 'has_discussions', 'forks_count',
'mirror_url', 'archived', 'disabled', 'open_issues_count',
'allow_forking', 'is_template', 'web_commit_signoff_required',
'topics', 'visibility', 'default_branch', 'allow_squash_merge',
'allow_merge_commit', 'allow_rebase_merge', 'allow_auto_merge',
'delete_branch_on_merge', 'allow_update_branch', 'use_squash_pr_title_as_default',
'squash_merge_commit_message', 'squash_merge_commit_title', 'merge_commit_message',
'merge_commit_title', 'network_count', 'subscribers_count'
]
})
// Additional fields that have different names in the database
mappedData.github_repo_id = data.id
mappedData.github_created_at = data.created_at
mappedData.github_updated_at = data.updated_at
mappedData.github_archived_at = data.archived_at
// license
mappedData.license_key = data.license?.key
mappedData.license_name = data.license?.name
mappedData.license_spdx_id = data.license?.spdx_id
mappedData.license_url = data.license?.url
mappedData.license_node_id = data.license?.node_id
// Security and analysis
mappedData.secret_scanning_status = data.security_and_analysis?.secret_scanning?.status
mappedData.secret_scanning_push_protection_status = data.security_and_analysis?.secret_scanning_push_protection?.status
mappedData.dependabot_security_updates_status = data.security_and_analysis?.dependabot_security_updates?.status
mappedData.secret_scanning_non_provider_patterns_status = data.security_and_analysis?.secret_scanning_non_provider_patterns?.status
mappedData.secret_scanning_validity_checks_status = data.security_and_analysis?.secret_scanning_validity_checks?.status
return mappedData
}
}
}

module.exports = {
Expand Down
75 changes: 2 additions & 73 deletions src/workflows/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,80 +1,9 @@
const debug = require('debug')('workflows')
const { simplifyObject } = require('@ulisesgascon/simplify-object')
const { github } = require('../providers')
const { initializeStore } = require('../store')
const { logger } = require('../utils')
const { validateGithubOrg, validateGithubListOrgRepos, validateGithubRepository } = require('../schemas')

const mapOrgData = (data) => {
const mappedData = simplifyObject(data, {
// Relevant fields for organizations
include: [
'login', 'node_id', 'url', 'avatar_url',
'description', 'name', 'company', 'blog',
'location', 'twitter_username', 'is_verified',
'has_organization_projects', 'has_repository_projects',
'public_repos', 'public_gists', 'followers',
'following', 'html_url', 'total_private_repos',
'owned_private_repos', 'private_gists', 'disk_usage',
'collaborators', 'default_repository_permission',
'default_repository_permission', 'members_can_create_repositories',
'two_factor_requirement_enabled', 'members_allowed_repository_creation_type',
'members_can_create_public_repositories', 'members_can_create_private_repositories',
'members_can_create_internal_repositories', 'members_can_create_pages',
'members_can_create_public_pages', 'members_can_create_private_pages',
'members_can_fork_private_repositories', 'web_commit_signoff_required',
'deploy_keys_enabled_for_repositories', 'dependency_graph_enabled_for_new_repositories',
'dependabot_alerts_enabled_for_new_repositories', 'dependabot_security_updates_enabled_for_new_repositories',
'advanced_security_enabled_for_new_repositories', 'secret_scanning_enabled_for_new_repositories',
'secret_scanning_push_protection_enabled_for_new_repositories', 'secret_scanning_push_protection_custom_link',
'secret_scanning_push_protection_custom_link_enabled']
})
// Additional fields that have different names in the database
mappedData.github_org_id = data.id
mappedData.github_created_at = data.created_at
mappedData.github_updated_at = data.updated_at
mappedData.github_archived_at = data.archived_at
return mappedData
}

const mapRepoData = (data) => {
const mappedData = simplifyObject(data, {
include: [
'node_id', 'name', 'full_name',
'html_url', 'description', 'fork', 'url',
'git_url', 'ssh_url', 'clone_url', 'svn_url',
'homepage', 'size', 'stargazers_count', 'watchers_count',
'language', 'has_issues', 'has_projects', 'has_downloads',
'has_wiki', 'has_pages', 'has_discussions', 'forks_count',
'mirror_url', 'archived', 'disabled', 'open_issues_count',
'allow_forking', 'is_template', 'web_commit_signoff_required',
'topics', 'visibility', 'default_branch', 'allow_squash_merge',
'allow_merge_commit', 'allow_rebase_merge', 'allow_auto_merge',
'delete_branch_on_merge', 'allow_update_branch', 'use_squash_pr_title_as_default',
'squash_merge_commit_message', 'squash_merge_commit_title', 'merge_commit_message',
'merge_commit_title', 'network_count', 'subscribers_count'
]
})
// Additional fields that have different names in the database
mappedData.github_repo_id = data.id
mappedData.github_created_at = data.created_at
mappedData.github_updated_at = data.updated_at
mappedData.github_archived_at = data.archived_at
// license
mappedData.license_key = data.license?.key
mappedData.license_name = data.license?.name
mappedData.license_spdx_id = data.license?.spdx_id
mappedData.license_url = data.license?.url
mappedData.license_node_id = data.license?.node_id
// Security and analysis
mappedData.secret_scanning_status = data.security_and_analysis?.secret_scanning?.status
mappedData.secret_scanning_push_protection_status = data.security_and_analysis?.secret_scanning_push_protection?.status
mappedData.dependabot_security_updates_status = data.security_and_analysis?.dependabot_security_updates?.status
mappedData.secret_scanning_non_provider_patterns_status = data.security_and_analysis?.secret_scanning_non_provider_patterns?.status
mappedData.secret_scanning_validity_checks_status = data.security_and_analysis?.secret_scanning_validity_checks?.status
return mappedData
}

const updateGithubOrgs = async (knex) => {
const { getAllGithubOrganizations, updateGithubOrganization } = initializeStore(knex)
const organizations = await getAllGithubOrganizations()
Expand All @@ -89,7 +18,7 @@ const updateGithubOrgs = async (knex) => {
debug('Validating data')
validateGithubOrg(data)
debug('Transforming data')
const mappedData = mapOrgData(data)
const mappedData = github.mappers.org(data)
debug('Updating organization in database')
await updateGithubOrganization(mappedData)
}))
Expand Down Expand Up @@ -121,7 +50,7 @@ const upsertGithubRepositories = async (knex) => {
debug(`Validating repository (${repo.full_name})`)
validateGithubRepository(repoData)
debug(`Transforming repository (${repo.full_name}) data`)
const mappedData = mapRepoData(repoData)
const mappedData = github.mappers.repo(repoData)
debug(`Upserting repository (${repo.full_name})`)
await upsertGithubRepository(mappedData, org.id)
}))
Expand Down

0 comments on commit 0e91fd0

Please sign in to comment.