chore: add new endpoint for backfill

spa/src/api/subscriptions/index.ts

@@ -1,7 +1,10 @@
 import { axiosRest } from "../axiosInstance";
+import { RestSyncReqBody } from "~/src/rest-interfaces";
 
 export default {
 	getSubscriptions: () => axiosRest.get("/rest/subscriptions"),
 	deleteSubscription: (subscriptionId: number) =>
-		axiosRest.delete(`/rest/app/cloud/subscriptions/${subscriptionId}`)
+		axiosRest.delete(`/rest/app/cloud/subscriptions/${subscriptionId}`),
+	syncSubscriptions: (subscriptionId: number, reqBody: RestSyncReqBody) =>
+		axiosRest.post(`/rest/app/cloud/subscriptions/${subscriptionId}/sync`, reqBody),
 };

src/rest-interfaces/index.ts

@@ -1,3 +1,8 @@
+export type RestSyncReqBody = {
+	syncType: string;
+	source: string;
+	commitsFromDate: string;
+}
 
 export type GetRedirectUrlResponse = {
 	redirectUrl: string;

src/rest/routes/subscriptions/index.ts

@@ -5,6 +5,7 @@ import { Installation } from "models/installation";
 import { removeSubscription } from "utils/jira-utils";
 import { GitHubServerApp } from "models/github-server-app";
 import { InvalidArgumentError } from "config/errors";
+import { SyncRouterHandler } from "./sync";
 
 export const SubscriptionsRouter = Router({ mergeParams: true  });
 
@@ -38,3 +39,5 @@ SubscriptionsRouter.delete("/", errorWrapper("SubscriptionDelete", async (req: R
 
 	res.sendStatus(204);
 }));
+
+SubscriptionsRouter.post("/sync", SyncRouterHandler);

src/rest/routes/subscriptions/sync.test.ts

@@ -0,0 +1,150 @@
+import { getFrontendApp } from "~/src/app";
+import { Installation } from "models/installation";
+import { Subscription } from "models/subscription";
+import express, { Express } from "express";
+import { RootRouter } from "routes/router";
+import supertest from "supertest";
+import { encodeSymmetric } from "atlassian-jwt";
+import { GitHubServerApp } from "models/github-server-app";
+import { v4 as newUUID } from "uuid";
+import { sqsQueues } from "~/src/sqs/queues";
+import { DatabaseStateCreator } from "~/test/utils/database-state-creator";
+
+jest.mock("~/src/sqs/queues");
+jest.mock("config/feature-flags");
+
+describe("Checking the deferred request parsing route", () => {
+	let app: Express;
+	let installation: Installation;
+	const installationIdForCloud = 1;
+	const installationIdForServer = 2;
+	const gitHubInstallationId = 15;
+	let subscription;
+	let gitHubServerApp: GitHubServerApp;
+	// let jwt: string;
+	const testSharedSecret = "test-secret";
+	const clientKey =  "jira-client-key";
+	const getToken = ({
+		secret = testSharedSecret,
+		iss = clientKey,
+		exp = Date.now() / 1000 + 10000,
+		qsh = "context-qsh",
+		sub = "myAccount" } = {}): string => {
+		return encodeSymmetric({
+			qsh,
+			iss,
+			exp,
+			sub
+		}, secret);
+	};
+	beforeEach(async () => {
+		app = getFrontendApp();
+		installation = await Installation.install({
+			host: jiraHost,
+			sharedSecret: testSharedSecret,
+			clientKey: clientKey
+		});
+		await Subscription.install({
+			installationId: installationIdForCloud,
+			host: jiraHost,
+			hashedClientKey: installation.clientKey,
+			gitHubAppId: undefined
+		});
+		gitHubServerApp = await GitHubServerApp.install({
+			uuid: newUUID(),
+			appId: 123,
+			gitHubAppName: "My GitHub Server App",
+			gitHubBaseUrl: gheUrl,
+			gitHubClientId: "lvl.1234",
+			gitHubClientSecret: "myghsecret",
+			webhookSecret: "mywebhooksecret",
+			privateKey: "myprivatekey",
+			installationId: installation.id
+		}, jiraHost);
+		await Subscription.install({
+			installationId: installationIdForServer,
+			host: jiraHost,
+			hashedClientKey: installation.clientKey,
+			gitHubAppId: gitHubServerApp.id
+		});
+		app = express();
+		app.use(RootRouter);
+		subscription = await Subscription.create({
+			gitHubInstallationId,
+			jiraHost
+		});
+	});
+
+	describe("cloud", () => {
+		it("should throw 401 error when no github token is passed", async () => {
+			const resp = await supertest(app)
+				.get(`/rest/app/cloud/subscriptions/${subscription.id}/sync`);
+
+			expect(resp.status).toEqual(401);
+		});
+
+		it("should return 403 on correct sub id with different jiraHost", async () => {
+			const commitsFromDate = new Date(new Date().getTime() - 2000);
+			const result = await new DatabaseStateCreator()
+				.forJiraHost("https://another-one.atlassian.net")
+				.create();
+			return supertest(app)
+				.post(`/rest/app/cloud/subscriptions/${result.subscription.id}/sync`)
+				.set("authorization", `${getToken()}`)
+				.send({
+					jiraHost,
+					syncType: "full",
+					commitsFromDate
+				})
+				.expect(403);
+		});
+
+		it("should return 400 on incorrect commitsFromDate", async () => {
+			const commitsFromDate = new Date(new Date().getTime() - 2000);
+			return supertest(app)
+				.post(`/rest/app/cloud/subscriptions/${undefined}/sync`)
+				.set("authorization", `${getToken()}`)
+				.send({
+					jiraHost,
+					syncType: "full",
+					commitsFromDate
+				})
+				.expect(400);
+		});
+
+		it("should return 400 on incorrect installationIdForCloud", async () => {
+			const commitsFromDate = new Date(new Date().getTime() + 2000);
+			return supertest(app)
+				.post(`/rest/app/cloud/subscriptions/${subscription.id}/sync`)
+				.set("authorization", `${getToken()}`)
+				.send({
+					jiraHost,
+					syncType: "full",
+					commitsFromDate
+				})
+				.expect(400);
+		});
+
+		it("should return 202 on correct post for /rest/app/cloud/sync one for Cloud app", async () => {
+			const commitsFromDate = new Date(new Date().getTime() - 2000);
+			return supertest(app)
+				.post(`/rest/app/cloud/subscriptions/${subscription.id}/sync`)
+				.set("authorization", `${getToken()}`)
+				.send({
+					jiraHost,
+					syncType: "full",
+					commitsFromDate
+				})
+				.expect(202)
+				.then(() => {
+					expect(sqsQueues.backfill.sendMessage).toBeCalledWith(expect.objectContaining({
+						jiraHost,
+						startTime: expect.anything(),
+						gitHubAppConfig: expect.objectContaining({ gitHubAppId: undefined, uuid: undefined })
+					}), expect.anything(), expect.anything());
+				});
+		});
+
+	});
+
+});

src/rest/routes/subscriptions/sync.ts

@@ -0,0 +1,99 @@
+import { Request, Response } from "express";
+import { ParamsDictionary } from "express-serve-static-core";
+import { errorWrapper } from "../../helper";
+import { Subscription } from "models/subscription";
+import { findOrStartSync } from "~/src/sync/sync-utils";
+import { determineSyncTypeAndTargetTasks } from "~/src/util/github-sync-helper";
+import { BaseLocals } from "..";
+import { InsufficientPermissionError, RestApiError } from "~/src/config/errors";
+import { RestSyncReqBody } from "~/src/rest-interfaces";
+// import { GitHubServerApp } from "~/src/models/github-server-app";
+
+const restSyncPost = async (
+	req: Request<ParamsDictionary, unknown, RestSyncReqBody>,
+	res: Response<string, BaseLocals>
+) => {
+	const {
+		syncType: syncTypeFromReq,
+		source,
+		commitsFromDate: commitsFrmDate
+	} = req.body;
+
+	// A date to start fetching commit history(main and branch) from.
+	const commitsFromDate = commitsFrmDate ? new Date(commitsFrmDate) : undefined;
+	if (commitsFromDate && commitsFromDate.valueOf() > Date.now()) {
+		throw new RestApiError(
+			400,
+			"INVALID_OR_MISSING_ARG",
+			"Invalid date value, cannot select a future date"
+		);
+	}
+
+	const subscriptionId: number = Number(req.params.subscriptionId);
+	if (!subscriptionId) {
+		req.log.info(
+			{
+				jiraHost: res.locals.installation.jiraHost,
+				subscriptionId
+			},
+			"Subscription ID not found when retrying sync."
+		);
+		throw new RestApiError(
+			400,
+			"INVALID_OR_MISSING_ARG",
+			"Subscription ID not found when retrying sync."
+		);
+	}
+
+	//TODO: We are yet to handle enterprise backfill
+	// const gitHubAppId: number | undefined = undefined;
+	// const cloudOrUUID = req.params.cloudOrUUID;
+	// const gheUUID = cloudOrUUID === "cloud" ? undefined : req.params.cloudOrUUID;
+	// if (gheUUID) {
+	// 	const ghEnterpriseServers: GitHubServerApp[] = await GitHubServerApp.findForInstallationId(gitHubInstallationId) || [];
+	// 	gitHubAppId = ghEnterpriseServers[0]?.appId;
+	// }
+
+	const subscription = await Subscription.findByPk(subscriptionId);
+
+	if (!subscription) {
+		req.log.info(
+			{
+				jiraHost: res.locals.installation.jiraHost,
+				subscriptionId
+			},
+			"Subscription not found when retrying sync."
+		);
+		throw new RestApiError(
+			400,
+			"INVALID_OR_MISSING_ARG",
+			"Subscription not found, cannot resync."
+		);
+	}
+
+	const localJiraHost = res.locals.installation.jiraHost;
+
+	if (subscription.jiraHost !== localJiraHost) {
+		throw new InsufficientPermissionError("Forbidden - mismatched Jira Host");
+	}
+
+
+	const { syncType, targetTasks } = determineSyncTypeAndTargetTasks(
+		syncTypeFromReq,
+		subscription
+	);
+	await findOrStartSync(
+		subscription,
+		req.log,
+		syncType,
+		commitsFromDate || subscription.backfillSince,
+		targetTasks,
+		{ source }
+	);
+	res.sendStatus(202);
+};
+
+export const SyncRouterHandler = errorWrapper(
+	"AnalyticsProxyHandler",
+	restSyncPost
+);

src/routes/jira/sync/jira-sync-post.ts

@@ -1,11 +1,11 @@
-import { Subscription, SyncStatus } from "models/subscription";
+import { Subscription } from "models/subscription";
 import * as Sentry from "@sentry/node";
 import { NextFunction, Request, Response } from "express";
 import { findOrStartSync } from "~/src/sync/sync-utils";
 import { sendAnalytics } from "utils/analytics-client";
 import { AnalyticsEventTypes, AnalyticsTrackEventsEnum, AnalyticsTrackSource } from "interfaces/common";
-import { TaskType, SyncType } from "~/src/sync/sync.types";
 import { booleanFlag, BooleanFlags } from "config/feature-flags";
+import { determineSyncTypeAndTargetTasks, getStartTimeInDaysAgo } from "../../../util/github-sync-helper";
 
 export const JiraSyncPost = async (req: Request, res: Response, next: NextFunction): Promise<void> => {
 	const { installationId: gitHubInstallationId, appId: gitHubAppId, syncType: syncTypeFromReq, source } = req.body;
@@ -35,7 +35,7 @@ export const JiraSyncPost = async (req: Request, res: Response, next: NextFuncti
 			return;
 		}
 
-		const { syncType, targetTasks } = await determineSyncTypeAndTargetTasks(syncTypeFromReq, subscription);
+		const { syncType, targetTasks } = determineSyncTypeAndTargetTasks(syncTypeFromReq, subscription);
 		await findOrStartSync(subscription, req.log, syncType, commitsFromDate || subscription.backfillSince, targetTasks, { source });
 
 		await sendAnalytics(res.locals.jiraHost, AnalyticsEventTypes.TrackEvent, {
@@ -64,26 +64,3 @@ export const JiraSyncPost = async (req: Request, res: Response, next: NextFuncti
 		next(new Error("Unauthorized"));
 	}
 };
-
-const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000;
-const getStartTimeInDaysAgo = (commitsFromDate: Date | undefined) => {
-	if (commitsFromDate === undefined) return undefined;
-	return Math.floor((Date.now() -  commitsFromDate?.getTime()) / MILLISECONDS_IN_ONE_DAY);
-};
-
-type SyncTypeAndTargetTasks = {
-	syncType: SyncType,
-	targetTasks: TaskType[] | undefined,
-};
-
-const determineSyncTypeAndTargetTasks = async (syncTypeFromReq: string, subscription: Subscription): Promise<SyncTypeAndTargetTasks> => {
-	if (syncTypeFromReq === "full") {
-		return { syncType: "full", targetTasks: undefined };
-	}
-
-	if (subscription.syncStatus === SyncStatus.FAILED) {
-		return { syncType: "full", targetTasks: undefined };
-	}
-
-	return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert", "codeScanningAlert"] };
-};

src/util/github-sync-helper.ts

@@ -0,0 +1,26 @@
+import { Subscription, SyncStatus } from "models/subscription";
+import { TaskType, SyncType } from "~/src/sync/sync.types";
+
+
+const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000;
+export const getStartTimeInDaysAgo = (commitsFromDate: Date | undefined) => {
+	if (commitsFromDate === undefined) return undefined;
+	return Math.floor((Date.now() -  commitsFromDate.getTime()) / MILLISECONDS_IN_ONE_DAY);
+};
+
+type SyncTypeAndTargetTasks = {
+	syncType: SyncType,
+	targetTasks: TaskType[] | undefined,
+};
+
+export const determineSyncTypeAndTargetTasks = (syncTypeFromReq: string, subscription: Subscription): SyncTypeAndTargetTasks => {
+	if (syncTypeFromReq === "full") {
+		return { syncType: "full", targetTasks: undefined };
+	}
+
+	if (subscription.syncStatus === SyncStatus.FAILED) {
+		return { syncType: "full", targetTasks: undefined };
+	}
+
+	return { syncType: "partial", targetTasks: ["pull", "branch", "commit", "build", "deployment", "dependabotAlert", "secretScanningAlert", "codeScanningAlert"] };
+};

test/snapshots/app.test.ts.snap

@@ -17,6 +17,8 @@ exports[`app getFrontendApp please review routes and update snapshot when adding
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionsGet
 :DELETE ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/?$
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionDelete
+:POST ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/sync/?$
+	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,AnalyticsProxyHandler
 :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/github-callback/?$
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,OAuthCallbackHandler
 :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/github-installed/?$
@@ -45,6 +47,8 @@ exports[`app getFrontendApp please review routes and update snapshot when adding
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionsGet
 :DELETE ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/?$
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionDelete
+:POST ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/sync/?$
+	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,AnalyticsProxyHandler
 :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/org/?(?=/|$)^/?$
 	query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,GitHubTokenHandler,GitHubOrgsFetchOrgs
 :POST ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/org/?(?=/|$)^/?$