From 80d7b1cdcd6b40e1ae54c7603d44c3ecf632924b Mon Sep 17 00:00:00 2001 From: gitcommitshow <56937085+gitcommitshow@users.noreply.github.com> Date: Wed, 13 Nov 2024 23:07:01 +0530 Subject: [PATCH 1/3] feat: snapshot cache to file --- .env.sample | 1 + README.md | 1 + app.js | 5 ++--- src/helpers.js | 16 +++++++++------- src/storage.js | 38 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 49 insertions(+), 12 deletions(-) diff --git a/.env.sample b/.env.sample index 402a84d..2f94fc7 100644 --- a/.env.sample +++ b/.env.sample @@ -2,6 +2,7 @@ WEBSITE_ADDRESS="https://github.app.home" LOGIN_USER=username LOGIN_PASSWORD=strongpassword DEFAULT_GITHUB_ORG=Git-Commit-Show +ONE_CLA_PER_ORG=true GITHUB_BOT_USERS=dependabot[bot],devops-github-rudderstack GITHUB_ORG_MEMBERS= APP_ID="11" diff --git a/README.md b/README.md index b692677..95c6888 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ A Node.js server for GitHub app to assist external contributors and save maintai - [x] On `rudder-transformer` PR merge, post a comment to raise PR in `integrations-config` - [ ] On `integrations-config` PR merge, psot a comment to join Slack's product-releases channel to get notified when that integration goes live - [ ] On `integrations-config` PR merge, post a comment to raise PR in `rudder-docs` +- [x] List of open PRs by external contributors ## Requirements diff --git a/app.js b/app.js index a5b3cdf..353c43c 100644 --- a/app.js +++ b/app.js @@ -1,4 +1,6 @@ import dotenv from "dotenv"; +// Load environment variables from .env file +dotenv.config(); import fs from "fs"; import http from "http"; import url from "url"; @@ -23,9 +25,6 @@ try { console.log(`Application version: ${APP_VERSION}`); console.log(`Website address: ${process.env.WEBSITE_ADDRESS}`); -// Load environment variables from .env file -dotenv.config(); - // Set configured values const appId = process.env.APP_ID; // To add GitHub App Private Key directly as a string config (instead of file), convert it to base64 by running following command diff --git a/src/helpers.js b/src/helpers.js index 8ea654f..f9ec41b 100644 --- a/src/helpers.js +++ b/src/helpers.js @@ -3,6 +3,8 @@ import { resolve } from "path"; import { PROJECT_ROOT_PATH } from "./config.js"; import url from "node:url"; +const ONE_CLA_PER_ORG = process.env.ONE_CLA_PER_ORG?.toLowerCase()?.trim() === "true"; + export function parseUrlQueryParams(urlString) { if(!urlString) return urlString; try{ @@ -79,15 +81,15 @@ export function isExternalContributionMaybe(pullRequest) { switch (pullRequest.author_association.toUpperCase()) { case "OWNER": pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, repo); + storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); return false; case "MEMBER": pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, repo); + storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); return false; case "COLLABORATOR": pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, repo); + storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); return false; default: //Will need more checks to verify author relation with the repo @@ -96,15 +98,15 @@ export function isExternalContributionMaybe(pullRequest) { } if (pullRequest?.head?.repo?.full_name !== pullRequest?.base?.repo?.full_name) { pullRequest.isExternalContribution = true; - storage.cache.set(true, username, "contribution", "external", owner, repo); + storage.cache.set(true, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); return true; } else if (pullRequest?.head?.repo?.full_name && pullRequest?.base?.repo?.full_name) { pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, repo); + storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); return false; } // Utilize cache if possible - const isConfirmedToBeExternalContributionInPast = storage.cache.get(username, "contribution", "external", owner, repo); + const isConfirmedToBeExternalContributionInPast = storage.cache.get(username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); if (typeof isConfirmedToBeExternalContributionInPast === "boolean") { pullRequest.isExternalContribution = isConfirmedToBeExternalContributionInPast; return isConfirmedToBeExternalContributionInPast @@ -126,7 +128,7 @@ async function isExternalContribution(octokit, pullRequest) { //TODO: Handle failure in checking permissions for the user const deterministicPermissionCheck = await isAllowedToWriteToTheRepo(octokit, username, owner, repo); pullRequest.isExternalContribution = deterministicPermissionCheck; - storage.cache.set(deterministicPermissionCheck, username, "contribution", "external", owner, repo); + storage.cache.set(deterministicPermissionCheck, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); return deterministicPermissionCheck; } diff --git a/src/storage.js b/src/storage.js index a6a1b45..d8c0554 100644 --- a/src/storage.js +++ b/src/storage.js @@ -3,8 +3,40 @@ import { resolve } from "path"; import { PROJECT_ROOT_PATH } from "./config.js"; const dbPath = process.env.DB_PATH || resolve(PROJECT_ROOT_PATH, "db.json"); +const cachePath = process.env.CACHE_PATH || resolve(PROJECT_ROOT_PATH, "cache.json"); createFileIfMissing(dbPath); -const CACHE = new Map(); +createFileIfMissing(cachePath); +const CACHE = initCache(); +let lastSnapshotTime = new Date().getTime(); +let pendingCacheToSnapshot = 0; +const CACHE_SNAPSHOT_INTERVAL = 1000 * 60 * 5; + +function initCache() { + try { + const json = fs.readFileSync(cachePath, 'utf-8'); // Read the file as a string + const obj = JSON.parse(json); // Parse JSON back to an object + return new Map(Object.entries(obj)); // Convert Object to a Map + } catch (err) { + return new Map(); + } +} + +async function lazyCacheSnapshot() { + try { + const currentTime = new Date().getTime(); + if ((currentTime - lastSnapshotTime) < CACHE_SNAPSHOT_INTERVAL) { + pendingCacheToSnapshot++; + return; + } + const obj = Object.fromEntries(CACHE); // Convert Map to an Object + const json = JSON.stringify(obj, null, 2); // Convert Object to JSON + fs.writeFile(cachePath, json, 'utf-8'); // Write JSON to a file + lastSnapshotTime = currentTime; + console.log(`Cache saved to ${cachePath}`); + } catch (err) { + console.error("Error in saving cache to file"); + } +} function createFileIfMissing(path) { try { @@ -51,7 +83,9 @@ export const storage = { }, set: function (value, ...args) { const key = args.join("/"); - return CACHE.set(key, value); + let cache = CACHE.set(key, value); + lazyCacheSnapshot(); + return cache } } }; From 9ee1a702afe14a95d06384797cbe49682ef5af5c Mon Sep 17 00:00:00 2001 From: gitcommitshow <56937085+gitcommitshow@users.noreply.github.com> Date: Thu, 14 Nov 2024 07:59:48 +0530 Subject: [PATCH 2/3] fix: store cache in file --- src/helpers.js | 18 ++++++++++-------- src/storage.js | 14 ++++++++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/helpers.js b/src/helpers.js index f9ec41b..f64fbfa 100644 --- a/src/helpers.js +++ b/src/helpers.js @@ -3,7 +3,9 @@ import { resolve } from "path"; import { PROJECT_ROOT_PATH } from "./config.js"; import url from "node:url"; -const ONE_CLA_PER_ORG = process.env.ONE_CLA_PER_ORG?.toLowerCase()?.trim() === "true"; +function isOneCLAPerOrgEnough() { + return process.env.ONE_CLA_PER_ORG?.toLowerCase()?.trim() === "true" ? true : false; +} export function parseUrlQueryParams(urlString) { if(!urlString) return urlString; @@ -81,15 +83,15 @@ export function isExternalContributionMaybe(pullRequest) { switch (pullRequest.author_association.toUpperCase()) { case "OWNER": pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + storage.cache.set(false, username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); return false; case "MEMBER": pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + storage.cache.set(false, username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); return false; case "COLLABORATOR": pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + storage.cache.set(false, username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); return false; default: //Will need more checks to verify author relation with the repo @@ -98,15 +100,15 @@ export function isExternalContributionMaybe(pullRequest) { } if (pullRequest?.head?.repo?.full_name !== pullRequest?.base?.repo?.full_name) { pullRequest.isExternalContribution = true; - storage.cache.set(true, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + storage.cache.set(true, username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); return true; } else if (pullRequest?.head?.repo?.full_name && pullRequest?.base?.repo?.full_name) { pullRequest.isExternalContribution = false; - storage.cache.set(false, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + storage.cache.set(false, username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); return false; } // Utilize cache if possible - const isConfirmedToBeExternalContributionInPast = storage.cache.get(username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + const isConfirmedToBeExternalContributionInPast = storage.cache.get(username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); if (typeof isConfirmedToBeExternalContributionInPast === "boolean") { pullRequest.isExternalContribution = isConfirmedToBeExternalContributionInPast; return isConfirmedToBeExternalContributionInPast @@ -128,7 +130,7 @@ async function isExternalContribution(octokit, pullRequest) { //TODO: Handle failure in checking permissions for the user const deterministicPermissionCheck = await isAllowedToWriteToTheRepo(octokit, username, owner, repo); pullRequest.isExternalContribution = deterministicPermissionCheck; - storage.cache.set(deterministicPermissionCheck, username, "contribution", "external", owner, ONE_CLA_PER_ORG ? undefined : repo); + storage.cache.set(deterministicPermissionCheck, username, "contribution", "external", owner, isOneCLAPerOrgEnough() ? undefined : repo); return deterministicPermissionCheck; } diff --git a/src/storage.js b/src/storage.js index d8c0554..2b8769b 100644 --- a/src/storage.js +++ b/src/storage.js @@ -8,7 +8,7 @@ createFileIfMissing(dbPath); createFileIfMissing(cachePath); const CACHE = initCache(); let lastSnapshotTime = new Date().getTime(); -let pendingCacheToSnapshot = 0; +let cacheSnapshotSize = CACHE.size; const CACHE_SNAPSHOT_INTERVAL = 1000 * 60 * 5; function initCache() { @@ -24,13 +24,19 @@ function initCache() { async function lazyCacheSnapshot() { try { const currentTime = new Date().getTime(); - if ((currentTime - lastSnapshotTime) < CACHE_SNAPSHOT_INTERVAL) { - pendingCacheToSnapshot++; + if ((currentTime - lastSnapshotTime) < CACHE_SNAPSHOT_INTERVAL || CACHE.size === cacheSnapshotSize) { return; } const obj = Object.fromEntries(CACHE); // Convert Map to an Object const json = JSON.stringify(obj, null, 2); // Convert Object to JSON - fs.writeFile(cachePath, json, 'utf-8'); // Write JSON to a file + fs.writeFile(cachePath, json, 'utf-8', function (err) { + if (!err) { + cacheSnapshotSize = CACHE.size; + console.log("Cache saved to file successfully. Total entries: " + cacheSnapshotSize); + } else { + console.error("Unexpected error in saving cache to file. Could be permission related issue."); + } + }); // Write JSON to a file lastSnapshotTime = currentTime; console.log(`Cache saved to ${cachePath}`); } catch (err) { From 1ab0431b3c081f12c9377f5ba971034cca3dd772 Mon Sep 17 00:00:00 2001 From: gitcommitshow <56937085+gitcommitshow@users.noreply.github.com> Date: Thu, 14 Nov 2024 08:23:13 +0530 Subject: [PATCH 3/3] feat: endpoint to reset contributions cached data --- app.js | 3 +++ src/routes.js | 6 ++++++ src/storage.js | 14 ++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/app.js b/app.js index 353c43c..adf71b3 100644 --- a/app.js +++ b/app.js @@ -241,6 +241,9 @@ http case "GET /contributions/pr": routes.getPullRequestDetail(req, res, app); break; + case "GET /contributions/reset": + routes.resetContributionData(req, res, app); + break; case "POST /api/webhook": middleware(req, res); break; diff --git a/src/routes.js b/src/routes.js index 97833ba..8bc6baf 100644 --- a/src/routes.js +++ b/src/routes.js @@ -221,6 +221,7 @@ export const routes = {

`); }, + resetContributionData(req, res, app) { + storage.cache.clear(); + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.write('Cache cleared'); + }, // ${!Array.isArray(prs) || prs?.length < 1 ? "No contributions found! (Might be an access issue)" : prs?.map(pr => `
  • ${pr?.user?.login} contributed a PR - ${pr?.title} [${pr?.labels?.map(label => label?.name).join('] [')}] updated ${timeAgo(pr?.updated_at)}
    • `).join('')} default(req, res) { res.writeHead(404); diff --git a/src/storage.js b/src/storage.js index 2b8769b..e00503a 100644 --- a/src/storage.js +++ b/src/storage.js @@ -21,6 +21,17 @@ function initCache() { } } +function clearCache() { + CACHE.clear(); + fs.truncate(cachePath, 0, (err) => { + if (err) { + console.error('Error truncating cache file:', err); + } else { + console.log('Cache file content deleted successfully.'); + } + }); +} + async function lazyCacheSnapshot() { try { const currentTime = new Date().getTime(); @@ -92,6 +103,9 @@ export const storage = { let cache = CACHE.set(key, value); lazyCacheSnapshot(); return cache + }, + clear: function () { + clearCache(); } } };