From 1dbad9651168c1bfdda5bbe26968947456927b2a Mon Sep 17 00:00:00 2001 From: Alex Birdsall Date: Thu, 17 Nov 2022 11:47:24 -0800 Subject: [PATCH] :window: :tada: Select dbt jobs with dropdown (#19502) * Add wrapper for cloud dbt endpoint Also includes a cheeky little test usage which probably should have had a name starting with an underscore (I did not commit the test code which added the new variable's contents to `(window as any).availableJobs`, on grounds that it was very ugly, but I did want to have the import and call of the wrapper function hit the git history here). * Add dbt Cloud jobs via dropdown (WIP: breaks if no integration) Selecting job from dropdown adds it to saved jobs, but the new endpoint is unconditionally called even though it will always throw an error for users with no dbt Cloud integration configured. * Refactor to stop errors in non-dbt-integrated workspaces Well, I suppose throwing a runtime error for every connection which could support dbt Cloud jobs but is part of a workspace with no integration set up, but that doesn't exactly seem ideal. Instead, this pulls all logic out of the top-level card except for pulling the dbt Cloud integration information; then it delegates the rest to either of two fairly self-contained components, `NoDbtIntegration` or the new `DbtJobsForm`. The immediate benefit is that I have a nice component boundary in which I can unconditionally run dbt-Cloud-only logic to fetch available jobs. * Filter already-selected jobs out of dropdown * Use dbt's jobNames and read-only {account,job}Id in job list * Remove obsolete yup validations for dbt Cloud jobs Since the values are now supplied by dbt Cloud via API, the user no longer has to manually input anything; and this sudden lack of user input rather obviates the need to validate user input. * Add button loading state when saving dbt Cloud jobs --- .../packages/cloud/lib/domain/dbtCloud/api.ts | 55 +++++ .../cloud/lib/domain/dbtCloud/index.ts | 1 + .../src/packages/cloud/services/dbtCloud.ts | 126 +++++++---- .../DbtCloudTransformationsCard.module.scss | 14 +- .../DbtCloudTransformationsCard.tsx | 199 ++++++++++-------- 5 files changed, 260 insertions(+), 135 deletions(-) create mode 100644 airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts create mode 100644 airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts diff --git a/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts new file mode 100644 index 000000000000..6c9a37a85679 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts @@ -0,0 +1,55 @@ +import { apiOverride } from "core/request/apiOverride"; + +/** + * Get the available dbt Cloud jobs associated with the given workspace config. + */ +export interface WorkspaceGetDbtJobsRequest { + workspaceId: WorkspaceId; + /** The config id associated with the dbt Cloud config, references the webhookConfigId in the core API. */ + dbtConfigId: string; +} + +/** + * The available dbt Cloud jobs for the requested workspace config + */ +export interface WorkspaceGetDbtJobsResponse { + availableDbtJobs: DbtCloudJobInfo[]; +} + +/** + * A dbt Cloud job + */ +export interface DbtCloudJobInfo { + /** The account id associated with the job */ + accountId: number; + /** The the specific job id returned by the dbt Cloud API */ + jobId: number; + /** The human-readable name of the job returned by the dbt Cloud API */ + jobName: string; +} + +/** + * @summary Calls the dbt Cloud `List Accounts` and `List jobs` APIs to get the list of available jobs for the dbt auth token associated with the requested workspace config. + */ +export const webBackendGetAvailableDbtJobsForWorkspace = ( + workspaceGetDbtJobsRequest: WorkspaceGetDbtJobsRequest, + options?: SecondParameter +) => { + return apiOverride( + { + url: `/v1/web_backend/cloud_workspaces/get_available_dbt_jobs`, + method: "post", + headers: { "Content-Type": "application/json" }, + data: workspaceGetDbtJobsRequest, + }, + options + ); +}; + +/** + * Workspace Id from OSS Airbyte instance + */ +export type WorkspaceId = string; + +// eslint-disable-next-line +type SecondParameter any> = T extends (config: any, args: infer P) => any ? P : never; diff --git a/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts new file mode 100644 index 000000000000..d158c5764011 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts @@ -0,0 +1 @@ +export * from "./api"; diff --git a/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts b/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts index 4504a91220a7..7ddfe49e124c 100644 --- a/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts +++ b/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts @@ -9,18 +9,28 @@ // - custom domains aren't yet supported import isEmpty from "lodash/isEmpty"; -import { useMutation } from "react-query"; +import { useMutation, useQuery } from "react-query"; import { OperatorType, WebBackendConnectionRead, OperationRead, WebhookConfigRead } from "core/request/AirbyteClient"; import { useWebConnectionService } from "hooks/services/useConnectionHook"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; +import { + DbtCloudJobInfo, + webBackendGetAvailableDbtJobsForWorkspace, + WorkspaceGetDbtJobsResponse, +} from "packages/cloud/lib/domain/dbtCloud/api"; +import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { useUpdateWorkspace } from "services/workspaces/WorkspacesService"; +import { useConfig } from "./config"; + export interface DbtCloudJob { account: string; job: string; operationId?: string; + jobName?: string; } +export type { DbtCloudJobInfo } from "packages/cloud/lib/domain/dbtCloud/api"; const dbtCloudDomain = "https://cloud.getdbt.com"; const webhookConfigName = "dbt cloud"; const executionBody = `{"cause": "airbyte"}`; @@ -28,26 +38,35 @@ const jobName = (t: DbtCloudJob) => `${t.account}/${t.job}`; const isDbtWebhookConfig = (webhookConfig: WebhookConfigRead) => !!webhookConfig.name?.includes("dbt"); -const toDbtCloudJob = (operation: OperationRead): DbtCloudJob => { - const { operationId } = operation; - const { executionUrl } = operation.operatorConfiguration.webhook || {}; +export const toDbtCloudJob = (operationOrCloudJob: OperationRead | DbtCloudJobInfo): DbtCloudJob => { + if ("operationId" in operationOrCloudJob) { + const { operationId } = operationOrCloudJob; + const { executionUrl } = operationOrCloudJob.operatorConfiguration.webhook || {}; - const matches = (executionUrl || "").match(/\/accounts\/([^/]+)\/jobs\/([^]+)\/run/); - if (!matches) { - throw new Error(`Cannot extract dbt cloud job params from executionUrl ${executionUrl}`); - } else { - const [, account, job] = matches; + const matches = (executionUrl || "").match(/\/accounts\/([^/]+)\/jobs\/([^]+)\/run/); + if (!matches) { + throw new Error(`Cannot extract dbt cloud job params from executionUrl ${executionUrl}`); + } else { + const [, account, job] = matches; - return { - account, - job, - operationId, - }; + return { + account, + job, + operationId, + }; + } + } else { + const { accountId, jobId, jobName } = operationOrCloudJob; + return { account: `${accountId}`, job: `${jobId}`, jobName }; } }; + const isDbtCloudJob = (operation: OperationRead): boolean => operation.operatorConfiguration.operatorType === OperatorType.webhook; +export const isSameJob = (remoteJob: DbtCloudJobInfo, savedJob: DbtCloudJob): boolean => + savedJob.account === `${remoteJob.accountId}` && savedJob.job === `${remoteJob.jobId}`; + export const useSubmitDbtCloudIntegrationConfig = () => { const { workspaceId } = useCurrentWorkspace(); const { mutateAsync: updateWorkspace } = useUpdateWorkspace(); @@ -78,35 +97,64 @@ export const useDbtIntegration = (connection: WebBackendConnectionRead) => { ); const otherOperations = [...(connection.operations?.filter((operation) => !isDbtCloudJob(operation)) || [])]; - const saveJobs = (jobs: DbtCloudJob[]) => { - // TODO dynamically use the workspace's configured dbt cloud domain when it gets returned by backend - const urlForJob = (job: DbtCloudJob) => `${dbtCloudDomain}/api/v2/accounts/${job.account}/jobs/${job.job}/run/`; - - return connectionService.update({ - connectionId: connection.connectionId, - operations: [ - ...otherOperations, - ...jobs.map((job) => ({ - workspaceId, - ...(job.operationId ? { operationId: job.operationId } : {}), - name: jobName(job), - operatorConfiguration: { - operatorType: OperatorType.webhook, - webhook: { - executionUrl: urlForJob(job), - // if `hasDbtIntegration` is true, webhookConfigId is guaranteed to exist - ...(webhookConfigId ? { webhookConfigId } : {}), - executionBody, + const { mutateAsync, isLoading } = useMutation({ + mutationFn: (jobs: DbtCloudJob[]) => { + // TODO dynamically use the workspace's configured dbt cloud domain when it gets returned by backend + const urlForJob = (job: DbtCloudJob) => `${dbtCloudDomain}/api/v2/accounts/${job.account}/jobs/${job.job}/run/`; + + return connectionService.update({ + connectionId: connection.connectionId, + operations: [ + ...otherOperations, + ...jobs.map((job) => ({ + workspaceId, + ...(job.operationId ? { operationId: job.operationId } : {}), + name: jobName(job), + operatorConfiguration: { + operatorType: OperatorType.webhook, + webhook: { + executionUrl: urlForJob(job), + // if `hasDbtIntegration` is true, webhookConfigId is guaranteed to exist + ...(webhookConfigId ? { webhookConfigId } : {}), + executionBody, + }, }, - }, - })), - ], - }); - }; + })), + ], + }); + }, + }); return { hasDbtIntegration, dbtCloudJobs, - saveJobs, + saveJobs: mutateAsync, + isSaving: isLoading, }; }; + +export const useAvailableDbtJobs = () => { + const { cloudApiUrl } = useConfig(); + const config = { apiUrl: cloudApiUrl }; + const middlewares = useDefaultRequestMiddlewares(); + const requestOptions = { config, middlewares }; + const workspace = useCurrentWorkspace(); + const { workspaceId } = workspace; + const dbtConfigId = workspace.webhookConfigs?.find((config) => config.name?.includes("dbt"))?.id; + + if (!dbtConfigId) { + throw new Error("cannot request available dbt jobs for a workspace with no dbt cloud integration configured"); + } + + const results = useQuery( + ["dbtCloud", dbtConfigId, "list"], + () => webBackendGetAvailableDbtJobsForWorkspace({ workspaceId, dbtConfigId }, requestOptions), + { + suspense: true, + } + ); + + // casting type to remove `| undefined`, since `suspense: true` will ensure the value + // is, in fact, available + return (results.data as WorkspaceGetDbtJobsResponse).availableDbtJobs; +}; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss index 98d2ce9c5f95..5cbf2fa69897 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss @@ -63,20 +63,20 @@ align-items: center; } -.jobListItemInputGroup { +.jobListItemIdFieldGroup { display: flex; justify-content: space-between; align-items: center; + flex-grow: 2; } -.jobListItemInput { +.jobListItemIdField { height: fit-content; margin-left: 1em; -} - -.jobListItemInputLabel { - font-size: 11px; - font-weight: 500; + background-color: colors.$grey-50; + flex-grow: 2; + padding: variables.$spacing-sm; + border-radius: variables.$border-radius-sm; } .jobListItemDelete { diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx index 37f3f88c577d..7dc068baa50e 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx @@ -1,21 +1,27 @@ import { faPlus, faXmark } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; -import { Field, Form, Formik, FieldArray, FieldProps, FormikHelpers } from "formik"; +import { Form, Formik, FieldArray, FormikHelpers } from "formik"; import { ReactNode } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Link } from "react-router-dom"; -import * as yup from "yup"; import { FormChangeTracker } from "components/common/FormChangeTracker"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; -import { Input } from "components/ui/Input"; +import { DropdownMenu } from "components/ui/DropdownMenu"; import { Text } from "components/ui/Text"; import { WebBackendConnectionRead } from "core/request/AirbyteClient"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; -import { DbtCloudJob, useDbtIntegration } from "packages/cloud/services/dbtCloud"; +import { + DbtCloudJob, + DbtCloudJobInfo, + isSameJob, + toDbtCloudJob, + useDbtIntegration, + useAvailableDbtJobs, +} from "packages/cloud/services/dbtCloud"; import { RoutePaths } from "pages/routePaths"; import dbtLogo from "./dbt-bit_tm.svg"; @@ -26,15 +32,6 @@ interface DbtJobListValues { jobs: DbtCloudJob[]; } -const dbtCloudJobListSchema = yup.object({ - jobs: yup.array().of( - yup.object({ - account: yup.number().required().positive().integer(), - job: yup.number().required().positive().integer(), - }) - ), -}); - export const DbtCloudTransformationsCard = ({ connection }: { connection: WebBackendConnectionRead }) => { // Possible render paths: // 1) IF the workspace has no dbt cloud account linked @@ -46,18 +43,67 @@ export const DbtCloudTransformationsCard = ({ connection }: { connection: WebBac // 2.2) AND the connection has saved dbt jobs // THEN show the jobs list and the "+ Add transformation" button - const { hasDbtIntegration, saveJobs, dbtCloudJobs } = useDbtIntegration(connection); + const { hasDbtIntegration, isSaving, saveJobs, dbtCloudJobs } = useDbtIntegration(connection); + + return hasDbtIntegration ? ( + + ) : ( + + ); +}; + +const NoDbtIntegration = () => { + const { workspaceId } = useCurrentWorkspace(); + const dbtSettingsPath = `/${RoutePaths.Workspaces}/${workspaceId}/${RoutePaths.Settings}/dbt-cloud`; + return ( + + + + } + > +
+ + {linkText}, + }} + /> + +
+ + ); +}; + +interface DbtJobsFormProps { + saveJobs: (jobs: DbtCloudJob[]) => Promise; + isSaving: boolean; + dbtCloudJobs: DbtCloudJob[]; +} +const DbtJobsForm: React.FC = ({ saveJobs, isSaving, dbtCloudJobs }) => { const onSubmit = (values: DbtJobListValues, { resetForm }: FormikHelpers) => { saveJobs(values.jobs).then(() => resetForm({ values })); }; + const availableDbtJobs = useAvailableDbtJobs(); + // because we don't store names for saved jobs, just the account and job IDs needed for + // webhook operation, we have to find the display names for saved jobs by comparing IDs + // with the list of available jobs as provided by dbt Cloud. + const jobs = dbtCloudJobs.map((savedJob) => { + const { jobName } = availableDbtJobs.find((remoteJob) => isSameJob(remoteJob, savedJob)) || {}; + const { account, job } = savedJob; + + return { account, job, jobName }; + }); + return ( { - return hasDbtIntegration ? ( + initialValues={{ jobs }} + render={({ values, dirty }) => { + return (
- + {() => ( + + )} + } > - + ); }} /> - ) : ( - - - - } - > - - ); }} /> ); }; -const DbtJobsList = ({ - jobs, - remove, - isValid, - dirty, -}: { +interface DbtJobsListProps { jobs: DbtCloudJob[]; remove: (i: number) => void; - isValid: boolean; dirty: boolean; -}) => ( + isLoading: boolean; +} + +const DbtJobsList = ({ jobs, remove, dirty, isLoading }: DbtJobsListProps) => (
{jobs.length ? ( <> - {jobs.map((_, i) => ( - remove(i)} /> + {jobs.map((job, i) => ( + remove(i)} /> ))} ) : ( @@ -131,46 +171,44 @@ const DbtJobsList = ({ -
); -// TODO give feedback on validation errors (red outline and validation message) -const JobsListItem = ({ jobIndex, removeJob }: { jobIndex: number; removeJob: () => void }) => { +interface JobsListItemProps { + job: DbtCloudJob; + removeJob: () => void; +} +const JobsListItem = ({ job, removeJob }: JobsListItemProps) => { const { formatMessage } = useIntl(); + // TODO if `job.jobName` is undefined, that means we failed to match any of the + // dbt-Cloud-supplied jobs with the saved job. This means one of two things has + // happened: + // 1) the user deleted the job in dbt Cloud, and we should make them delete it from + // their webhook operations. If we have a nonempty list of other dbt Cloud jobs, + // it's definitely this. + // 2) the API call to fetch the names failed somehow (possibly with a 200 status, if there's a bug) + const title = {job.jobName || formatMessage({ id: "connection.dbtCloudJobs.job.title" })}; + return (
- + {title}
-
-
- - {({ field }: FieldProps) => ( - <> - - - - )} - +
+
+ + {formatMessage({ id: "connection.dbtCloudJobs.job.accountId" })}: {job.account} +
-
- - {({ field }: FieldProps) => ( - <> - - - - )} - +
+ + {formatMessage({ id: "connection.dbtCloudJobs.job.jobId" })}: {job.job} +