diff --git a/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss b/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss index 7da33c2358ef..dd0761d0e766 100644 --- a/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss +++ b/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss @@ -36,11 +36,10 @@ $border-width: variables.$border-thick; white-space: nowrap; } -.dropdown { +.control { margin-left: auto; padding: 0 variables.$spacing-xs; background-color: colors.$white; - min-width: calc(50% - 100px); } .content { diff --git a/airbyte-webapp/src/components/GroupControls/GroupControls.tsx b/airbyte-webapp/src/components/GroupControls/GroupControls.tsx index 580e8c1fad1d..c69eb8bcbf6d 100644 --- a/airbyte-webapp/src/components/GroupControls/GroupControls.tsx +++ b/airbyte-webapp/src/components/GroupControls/GroupControls.tsx @@ -1,21 +1,29 @@ +import classNames from "classnames"; import React from "react"; import styles from "./GroupControls.module.scss"; interface GroupControlsProps { label: React.ReactNode; - dropdown?: React.ReactNode; + control?: React.ReactNode; + controlClassName?: string; name?: string; } -const GroupControls: React.FC> = ({ label, dropdown, children, name }) => { +const GroupControls: React.FC> = ({ + label, + control, + children, + name, + controlClassName, +}) => { return ( // This outer div is necessary for .content > :first-child padding to be properly applied in the case of nested GroupControls
{label}
-
{dropdown}
+
{control}
{children} diff --git a/airbyte-webapp/src/components/GroupControls/index.stories.tsx b/airbyte-webapp/src/components/GroupControls/index.stories.tsx index 3d99975add51..41c5266f1188 100644 --- a/airbyte-webapp/src/components/GroupControls/index.stories.tsx +++ b/airbyte-webapp/src/components/GroupControls/index.stories.tsx @@ -1,5 +1,6 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; +import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { FormBlock, FormConditionItem } from "core/form/types"; @@ -73,3 +74,21 @@ WithContent.args = { ), }; + +export const EmptyWithControl = Template.bind({}); +EmptyWithControl.args = { + label, + control: , +}; + +export const ControlAndContent = Template.bind({}); +ControlAndContent.args = { + label, + control: , + children: ( + <> + Content part 1 + Content part 2 + + ), +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 274ca47e84d0..1933579f2fc4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -1,6 +1,9 @@ import { Form, Formik, useField } from "formik"; +import merge from "lodash/merge"; import { useState } from "react"; +import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; +import * as yup from "yup"; import { Button } from "components/ui/Button"; import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; @@ -8,7 +11,7 @@ import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; import { FormikPatch } from "core/form/FormikPatch"; import { ReactComponent as PlusIcon } from "../../connection/ConnectionOnboarding/plusIcon.svg"; -import { BuilderStream } from "../types"; +import { BuilderStream, DEFAULT_BUILDER_STREAM_VALUES } from "../types"; import styles from "./AddStreamButton.module.scss"; import { BuilderField } from "./BuilderField"; @@ -19,44 +22,48 @@ interface AddStreamValues { interface AddStreamButtonProps { onAddStream: (addedStreamNum: number) => void; + button?: React.ReactElement; + initialValues?: Partial; } -export const AddStreamButton: React.FC = ({ onAddStream }) => { +export const AddStreamButton: React.FC = ({ onAddStream, button, initialValues }) => { const { formatMessage } = useIntl(); const [isOpen, setIsOpen] = useState(false); const [streamsField, , helpers] = useField("streams"); const numStreams = streamsField.value.length; + const buttonClickHandler = () => { + setIsOpen(true); + }; + return ( <> - + {isOpen &&
{children}
} +
+ ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index ee4927d9320c..1248a975e097 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -31,6 +31,7 @@ .connectorNameText { margin-left: auto; margin-right: auto; + margin-bottom: variables.$spacing-lg; } .streamsHeader { @@ -83,7 +84,7 @@ } .globalConfigButton { - margin-top: variables.$spacing-xl; + margin-top: variables.$spacing-sm; } .streamList { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 11a2f53b553e..7bd54599ea6b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,4 +1,4 @@ -import { faSliders } from "@fortawesome/free-solid-svg-icons"; +import { faSliders, faUser } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useFormikContext } from "formik"; @@ -10,14 +10,10 @@ import { Heading } from "components/ui/Heading"; import { Text } from "components/ui/Text"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { - BuilderView, - DEFAULT_BUILDER_FORM_VALUES, - useConnectorBuilderState, -} from "services/connectorBuilder/ConnectorBuilderStateService"; +import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "../DownloadYamlButton"; -import { BuilderFormValues } from "../types"; +import { BuilderFormValues, DEFAULT_BUILDER_FORM_VALUES, getInferredInputs } from "../types"; import { useBuilderErrors } from "../useBuilderErrors"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; @@ -76,7 +72,7 @@ export const BuilderSidebar: React.FC = ({ className, toggl }; const handleViewSelect = (selectedView: BuilderView) => { setSelectedView(selectedView); - if (selectedView !== "global") { + if (selectedView !== "global" && selectedView !== "inputs") { setTestStreamIndex(selectedView); } }; @@ -108,6 +104,19 @@ export const BuilderSidebar: React.FC = ({ className, toggl + handleViewSelect("inputs")} + > + + + +
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index d8495cd83000..bc8e29a532a5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -1,5 +1,6 @@ import { useIntl } from "react-intl"; +import { AuthenticationSection } from "./AuthenticationSection"; import { BuilderCard } from "./BuilderCard"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; @@ -14,8 +15,9 @@ export const GlobalConfigView: React.FC = () => { {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */} - + + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx new file mode 100644 index 000000000000..3d8607eb85f9 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx @@ -0,0 +1,49 @@ +import { useField } from "formik"; + +import { RequestOption } from "core/request/ConnectorManifest"; + +import { injectIntoValues } from "../types"; +import { BuilderField } from "./BuilderField"; + +interface InjectRequestOptionFieldsProps { + path: string; + descriptor: string; + excludeInjectIntoValues?: string[]; +} + +export const InjectRequestOptionFields: React.FC = ({ + path, + descriptor, + excludeInjectIntoValues, +}) => { + const [field, , helpers] = useField(path); + + return ( + <> + !excludeInjectIntoValues.includes(val)) + : injectIntoValues + } + onChange={(newValue) => { + if (newValue === "path") { + helpers.setValue({ inject_into: newValue, field_name: undefined, type: "RequestOption" }); + } + }} + label="Inject into" + tooltip={`Configures where the ${descriptor} should be set on the HTTP requests`} + /> + {field.value.inject_into !== "path" && ( + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss new file mode 100644 index 000000000000..36580839d42d --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss @@ -0,0 +1,50 @@ +@use "scss/variables"; +@use "scss/colors"; + +.list { + display: flex; + flex-direction: column; + gap: variables.$spacing-md; + list-style-type: none; + padding: 0; + align-items: stretch; + margin: 0; +} + +.listItem { + display: flex; + align-items: center; +} + +.itemLabel { + flex-grow: 1; +} + +.itemButton { + background: none !important; + border: none !important; + padding: variables.$spacing-xs; +} + +.inputForm { + gap: variables.$spacing-lg; + display: flex; + flex-direction: column; +} + +.inputsDescription { + margin-top: variables.$spacing-xl; + margin-bottom: variables.$spacing-lg; +} + +.addInputButton { + align-self: center; +} + +.inputsCard { + align-self: stretch; +} + +.deleteButtonContainer { + flex-grow: 1; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx new file mode 100644 index 000000000000..77095bd805fc --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -0,0 +1,352 @@ +import { faGear, faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Form, Formik, useField, useFormikContext } from "formik"; +import { JSONSchema7 } from "json-schema"; +import { useMemo, useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; +import { useEffectOnce } from "react-use"; +import * as yup from "yup"; + +import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; +import { InfoBox } from "components/ui/InfoBox"; +import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import { Text } from "components/ui/Text"; + +import { FormikPatch } from "core/form/FormikPatch"; + +import { BuilderFormInput, BuilderFormValues, getInferredInputs } from "../types"; +import { BuilderConfigView } from "./BuilderConfigView"; +import { BuilderField } from "./BuilderField"; +import styles from "./InputsView.module.scss"; + +const supportedTypes = ["string", "integer", "number", "array", "boolean", "enum", "unknown"] as const; + +interface InputInEditing { + key: string; + definition: JSONSchema7; + required: boolean; + isNew?: boolean; + showDefaultValueField: boolean; + type: typeof supportedTypes[number]; + isInferredInputOverride: boolean; +} + +function sluggify(str: string) { + return str.toLowerCase().replaceAll(/[^a-zA-Z\d]/g, "_"); +} + +function newInputInEditing(): InputInEditing { + return { + key: "", + definition: {}, + required: false, + isNew: true, + showDefaultValueField: false, + type: "string", + isInferredInputOverride: false, + }; +} + +function formInputToInputInEditing( + { key, definition, required }: BuilderFormInput, + isInferredInputOverride: boolean +): InputInEditing { + const supportedType = supportedTypes.find((type) => type === definition.type) || "unknown"; + return { + key, + definition, + required, + isNew: false, + showDefaultValueField: Boolean(definition.default), + type: supportedType !== "unknown" && definition.enum ? "enum" : supportedType, + isInferredInputOverride, + }; +} + +function inputInEditingToFormInput({ + type, + showDefaultValueField, + isNew, + ...values +}: InputInEditing): BuilderFormInput { + return { + ...values, + definition: { + ...values.definition, + type: type === "enum" ? "string" : type === "unknown" ? values.definition.type : type, + // only respect the enum values if the user explicitly selected enum as type + enum: type === "enum" && values.definition.enum?.length ? values.definition.enum : undefined, + default: showDefaultValueField ? values.definition.default : undefined, + }, + }; +} + +export const InputsView: React.FC = () => { + const { formatMessage } = useIntl(); + const { values, setFieldValue } = useFormikContext(); + const [inputs, , helpers] = useField("inputs"); + const [inputInEditing, setInputInEditing] = useState(undefined); + const inferredInputs = useMemo(() => getInferredInputs(values), [values]); + const usedKeys = useMemo( + () => [...inputs.value, ...inferredInputs].map((input) => input.key), + [inputs.value, inferredInputs] + ); + const inputInEditValidation = useMemo( + () => + yup.object().shape({ + // make sure key can only occur once + key: yup + .string() + .notOneOf( + inputInEditing?.isNew ? usedKeys : usedKeys.filter((key) => key !== inputInEditing?.key), + "connectorBuilder.duplicateFieldID" + ), + required: yup.bool(), + definition: yup.object().shape({ + title: yup.string().required("form.empty.error"), + }), + }), + [inputInEditing?.isNew, inputInEditing?.key, usedKeys] + ); + + return ( + + + + + {(inputs.value.length > 0 || inferredInputs.length > 0) && ( + +
    + {inferredInputs.map((input) => ( + + ))} + {inputs.value.map((input) => ( + + ))} +
+ + )} + + {inputInEditing && ( + { + if (values.isInferredInputOverride) { + setFieldValue(`inferredInputOverrides.${values.key}`, values.definition); + } else { + const newInput = inputInEditingToFormInput(values); + helpers.setValue( + inputInEditing.isNew + ? [...inputs.value, newInput] + : inputs.value.map((input) => (input.key === inputInEditing.key ? newInput : input)) + ); + } + setInputInEditing(undefined); + }} + > + <> + + { + helpers.setValue(inputs.value.filter((input) => input.key !== inputInEditing.key)); + setInputInEditing(undefined); + }} + onClose={() => { + setInputInEditing(undefined); + }} + /> + + + )} + + ); +}; + +const InputModal = ({ + inputInEditing, + onClose, + onDelete, +}: { + inputInEditing: InputInEditing; + onDelete: () => void; + onClose: () => void; +}) => { + const isInferredInputOverride = inputInEditing.isInferredInputOverride; + const { isValid, values, setFieldValue, setTouched } = useFormikContext(); + + const { formatMessage } = useIntl(); + useEffectOnce(() => { + // key input is always touched so errors are shown right away as it will be auto-set by the user changing the title + setTouched({ key: true }); + }); + + return ( + + } + onClose={onClose} + > +
+ + { + if (!isInferredInputOverride) { + setFieldValue("key", sluggify(newValue || ""), true); + } + }} + label={formatMessage({ id: "connectorBuilder.inputModal.inputName" })} + tooltip={formatMessage({ id: "connectorBuilder.inputModal.inputNameTooltip" })} + /> + + + {values.type !== "unknown" && !isInferredInputOverride ? ( + <> + { + setFieldValue("definition.default", undefined); + }} + label={formatMessage({ id: "connectorBuilder.inputModal.type" })} + tooltip={formatMessage({ id: "connectorBuilder.inputModal.typeTooltip" })} + /> + {values.type === "enum" && ( + + )} + + + + {values.showDefaultValueField && ( + + )} + + + ) : ( + + {isInferredInputOverride ? ( + + ) : ( + + )} + + )} + + + {!inputInEditing.isNew && !inputInEditing.isInferredInputOverride && ( +
+ +
+ )} + + + +
+ + ); +}; + +const InputItem = ({ + input, + setInputInEditing, + isInferredInput, +}: { + input: BuilderFormInput; + setInputInEditing: (inputInEditing: InputInEditing) => void; + isInferredInput: boolean; +}): JSX.Element => { + return ( +
  • +
    {input.definition.title || input.key}
    + +
    • + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx index d66655fe1981..487534424234 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx @@ -49,7 +49,14 @@ export const KeyValueListField: React.FC = ({ path, labe const [{ value: keyValueList }, , { setValue: setKeyValueList }] = useField>(path); return ( - }> + } + control={ + + } + > {keyValueList.map((keyValue, keyValueIndex) => ( = ({ path, labe }} /> ))} -
    - -
    ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx new file mode 100644 index 000000000000..0a6878c28fa0 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -0,0 +1,170 @@ +import { useField } from "formik"; + +import GroupControls from "components/GroupControls"; +import { ControlLabels } from "components/LabeledControl"; + +import { RequestOption } from "core/request/ConnectorManifest"; + +import { BuilderPaginator } from "../types"; +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; +import { BuilderOneOf } from "./BuilderOneOf"; +import { InjectRequestOptionFields } from "./InjectRequestOptionFields"; +import { ToggleGroupField } from "./ToggleGroupField"; + +interface PaginationSectionProps { + streamFieldPath: (fieldPath: string) => string; +} + +export const PaginationSection: React.FC = ({ streamFieldPath }) => { + const [field, , helpers] = useField(streamFieldPath("paginator")); + const [pageSizeField] = useField(streamFieldPath("paginator.strategy.page_size")); + const [, , pageSizeOptionHelpers] = useField(streamFieldPath("paginator.pageSizeOption")); + + const handleToggle = (newToggleValue: boolean) => { + if (newToggleValue) { + helpers.setValue({ + strategy: { + type: "OffsetIncrement", + page_size: "", + }, + pageTokenOption: { + type: "RequestOption", + inject_into: "request_parameter", + }, + }); + } else { + helpers.setValue(undefined); + } + }; + const toggledOn = field.value !== undefined; + + const pageTokenOption = ( + + } + > + + + ); + + const pageSizeOption = ( + + label="Page size option" + tooltip="Configures how the page size will be sent in requests to the source API" + fieldPath={streamFieldPath("paginator.pageSizeOption")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + ); + + return ( + + ), + toggledOn, + onToggle: handleToggle, + }} + > + + + {pageSizeOption} + {pageTokenOption} + + ), + }, + { + label: "Page Increment", + typeValue: "PageIncrement", + children: ( + <> + + + {pageSizeOption} + {pageTokenOption} + + ), + }, + { + label: "Cursor Pagination", + typeValue: "CursorPagination", + children: ( + <> + + + { + if (newValue === undefined || newValue === "") { + pageSizeOptionHelpers.setValue(undefined); + } + }} + label="Page size" + tooltip="Set the size of each page" + optional + /> + {pageSizeField.value && pageSizeField.value !== "" && pageSizeOption} + {pageTokenOption} + + ), + }, + ]} + /> + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss index f59cc288f5b9..bfb25201dc5e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss @@ -1,20 +1,21 @@ @use "scss/variables"; @use "scss/colors"; -$deleteButtonWidth: 24px; +$controlButtonWidth: 24px; .controls { display: flex; justify-content: center; + gap: variables.$spacing-sm; } -.deleteButton { +.controlButton { border: none; background-color: transparent; color: colors.$grey-400; border-radius: variables.$border-radius-xs; - width: $deleteButtonWidth; - height: $deleteButtonWidth; + width: $controlButtonWidth; + height: $controlButtonWidth; padding: variables.$spacing-xs; padding: 0; cursor: pointer; @@ -22,6 +23,12 @@ $deleteButtonWidth: 24px; &:hover { color: colors.$white; + background-color: colors.$grey-800; + } +} + +.deleteButton { + &:hover { background-color: colors.$red; } } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index d212d530f20e..1171aac969e4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -1,5 +1,6 @@ -import { faTrashCan } from "@fortawesome/free-regular-svg-icons"; +import { faTrashCan, faCopy } from "@fortawesome/free-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import classNames from "classnames"; import { useField } from "formik"; import { useIntl } from "react-intl"; @@ -7,12 +8,15 @@ import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { BuilderStream } from "../types"; +import { AddStreamButton } from "./AddStreamButton"; import { BuilderCard } from "./BuilderCard"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; import { BuilderTitle } from "./BuilderTitle"; import { KeyValueListField } from "./KeyValueListField"; +import { PaginationSection } from "./PaginationSection"; import styles from "./StreamConfigView.module.scss"; +import { StreamSlicerSection } from "./StreamSlicerSection"; interface StreamConfigViewProps { streamNum: number; @@ -49,13 +53,25 @@ export const StreamConfigView: React.FC = ({ streamNum }) {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */}
    - + } + /> +
    = ({ streamNum }) + + + string; +} + +export const StreamSlicerSection: React.FC = ({ streamFieldPath }) => { + const [field, , helpers] = useField(streamFieldPath("streamSlicer")); + + const handleToggle = (newToggleValue: boolean) => { + if (newToggleValue) { + helpers.setValue({ + type: "ListStreamSlicer", + slice_values: [], + cursor_field: "", + }); + } else { + helpers.setValue(undefined); + } + }; + const toggledOn = field.value !== undefined; + + return ( + + ), + toggledOn, + onToggle: handleToggle, + }} + > + + + + + label="Slice request option" + tooltip="Optionally configures how the slice values will be sent in requests to the source API" + fieldPath={streamFieldPath("streamSlicer.request_option")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + + ), + }, + { + label: "Datetime", + typeValue: "DatetimeStreamSlicer", + children: ( + <> + + + + + + + + + label="Start time request option" + tooltip="Optionally configures how the start datetime will be sent in requests to the source API" + fieldPath={streamFieldPath("streamSlicer.start_time_option")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + + label="End time request option" + tooltip="Optionally configures how the end datetime will be sent in requests to the source API" + fieldPath={streamFieldPath("streamSlicer.end_time_option")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + + + + + ), + }, + ]} + /> + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss new file mode 100644 index 000000000000..1f016d2cb288 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss @@ -0,0 +1,12 @@ +@use "scss/variables"; + +.label { + display: flex; + align-items: center; + gap: variables.$spacing-md; + height: 34px; + + label { + padding-bottom: 0; + } +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx new file mode 100644 index 000000000000..865b48ad2aae --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx @@ -0,0 +1,40 @@ +import { useField } from "formik"; + +import GroupControls from "components/GroupControls"; +import { ControlLabels } from "components/LabeledControl"; +import { CheckBox } from "components/ui/CheckBox"; + +import styles from "./ToggleGroupField.module.scss"; + +interface ToggleGroupFieldProps { + label: string; + tooltip: string; + fieldPath: string; + initialValues: T; +} + +// eslint-disable-next-line react/function-component-definition +export function ToggleGroupField({ + children, + label, + tooltip, + fieldPath, + initialValues, +}: React.PropsWithChildren>) { + const [field, , helpers] = useField(fieldPath); + const enabled = field.value !== undefined; + + const labelComponent = ( +
    + { + event.target.checked ? helpers.setValue(initialValues) : helpers.setValue(undefined); + }} + /> + +
    + ); + + return enabled ? {children} : labelComponent; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.module.scss new file mode 100644 index 000000000000..da549df1adae --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.module.scss @@ -0,0 +1,12 @@ +@use "scss/variables"; +@use "scss/colors"; + +.container { + display: flex; + justify-content: space-between; + align-items: center; +} + +.icon { + color: colors.$blue-400; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.tsx new file mode 100644 index 000000000000..80eb465e323e --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.tsx @@ -0,0 +1,18 @@ +import { faUser } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { FormattedMessage } from "react-intl"; + +import { ControlLabels } from "components/LabeledControl"; +import { Tooltip } from "components/ui/Tooltip"; + +import styles from "./UserInputField.module.scss"; + +export const UserInputField: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => { + return ( + + }> + + + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss index b9017a3447a5..73b43de5ec13 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss @@ -1,7 +1,41 @@ @use "scss/colors"; +@use "scss/variables"; -.modalContent { - height: 60vh; - overflow: visible; - background-color: colors.$grey-100; +.icon { + color: colors.$grey-500; +} + +.formContent { + max-height: 60vh; + overflow: auto; +} + +.inputFormModalFooter { + border-top: variables.$border-thin solid colors.$grey-100; + gap: variables.$spacing-md; + padding: 0 variables.$spacing-xl; + margin: 0 -1 * variables.$spacing-xl; +} + +.inputFormModalFooter > * { + // need to overwrite the margin of the button wrapper used within create controls + // TODO refactor so this isn't necessary + margin-top: variables.$spacing-lg !important; +} + +.warningBox { + margin-bottom: variables.$spacing-lg; + background-color: colors.$blue-50; +} + +.warningBoxContainer { + display: flex; + gap: variables.$spacing-md; + align-items: center; +} + +.inputsErrorBadge { + position: absolute; + top: -1 * variables.$spacing-md; + right: -1 * variables.$spacing-md; } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index b9685aad35b6..730bdfc44d60 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -1,54 +1,128 @@ -import { faGear } from "@fortawesome/free-solid-svg-icons"; +import { faClose, faUser } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { useState } from "react"; -import { FormattedMessage } from "react-intl"; +import { useMemo } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; +import { useLocalStorage } from "react-use"; import { Button } from "components/ui/Button"; -import { CodeEditor } from "components/ui/CodeEditor"; -import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import { InfoBox } from "components/ui/InfoBox"; +import { Modal, ModalBody } from "components/ui/Modal"; +import { NumberBadge } from "components/ui/NumberBadge"; +import { Tooltip } from "components/ui/Tooltip"; +import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; +import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { ConnectorForm } from "views/Connector/ConnectorForm"; import styles from "./ConfigMenu.module.scss"; +import { ConfigMenuErrorBoundaryComponent } from "./ConfigMenuErrorBoundary"; interface ConfigMenuProps { className?: string; + configJsonErrors: number; + isOpen: boolean; + setIsOpen: (open: boolean) => void; } -export const ConfigMenu: React.FC = ({ className }) => { - const [isOpen, setIsOpen] = useState(false); - const { configString, setConfigString } = useConnectorBuilderState(); +export const ConfigMenu: React.FC = ({ className, configJsonErrors, isOpen, setIsOpen }) => { + const { formatMessage } = useIntl(); + const { configJson, setConfigJson, jsonManifest, editorView, setEditorView } = useConnectorBuilderState(); + + const [showInputsWarning, setShowInputsWarning] = useLocalStorage("connectorBuilderInputsWarning", true); + + const switchToYaml = () => { + setEditorView("yaml"); + setIsOpen(false); + }; + + const connectorDefinitionSpecification: SourceDefinitionSpecificationDraft | undefined = useMemo( + () => + jsonManifest.spec + ? { + documentationUrl: jsonManifest.spec.documentation_url, + connectionSpecification: jsonManifest.spec.connection_specification, + } + : undefined, + [jsonManifest] + ); return ( <> - + {configJsonErrors > 0 && ( + + )} + + } + placement={editorView === "yaml" ? "left" : "top"} + containerClassName={className} + > + {jsonManifest.spec ? ( + + ) : editorView === "ui" ? ( + + ) : ( + + )} + + {isOpen && connectorDefinitionSpecification && ( setIsOpen(false)} title={} > - - { - setConfigString(val ?? ""); - }} - /> + + + <> + {showInputsWarning && ( + +
    +
    + +
    +
    +     + )} + { + setConfigJson(values.connectionConfiguration as StreamReadRequestBodyConfig); + setIsOpen(false); + }} + onCancel={() => { + setIsOpen(false); + }} + onReset={() => { + setConfigJson({}); + }} + submitLabel={formatMessage({ id: "connectorBuilder.saveInputsForm" })} + /> + +     - - -     )} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss new file mode 100644 index 000000000000..0cc75c2324e9 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss @@ -0,0 +1,9 @@ +@use "scss/colors"; +@use "scss/variables"; + +.errorContent { + display: flex; + flex-direction: column; + gap: variables.$spacing-lg; + align-items: flex-end; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx new file mode 100644 index 000000000000..972c92c90a2e --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx @@ -0,0 +1,66 @@ +import React from "react"; +import { FormattedMessage } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { InfoBox } from "components/ui/InfoBox"; + +import { FormBuildError, isFormBuildError } from "core/form/FormBuildError"; +import { EditorView } from "services/connectorBuilder/ConnectorBuilderStateService"; + +import styles from "./ConfigMenuErrorBoundary.module.scss"; + +interface ApiErrorBoundaryState { + error?: string | FormBuildError; +} + +interface ApiErrorBoundaryProps { + closeAndSwitchToYaml: () => void; + currentView: EditorView; +} + +export class ConfigMenuErrorBoundaryComponent extends React.Component< + React.PropsWithChildren, + ApiErrorBoundaryState +> { + state: ApiErrorBoundaryState = {}; + + static getDerivedStateFromError(error: { message: string; __type?: string }): ApiErrorBoundaryState { + if (isFormBuildError(error)) { + return { error }; + } + + return { error: error.message }; + } + render(): React.ReactNode { + const { children, currentView, closeAndSwitchToYaml } = this.props; + const { error } = this.state; + + if (!error) { + return children; + } + return ( +
    + + }} + />{" "} + + + + + +
    + ); + } +} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx index 482893c176e0..30fe955bbbff 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx @@ -14,19 +14,29 @@ import styles from "./StreamTestButton.module.scss"; interface StreamTestButtonProps { readStream: () => void; + hasConfigJsonErrors: boolean; + setTestInputOpen: (open: boolean) => void; } -export const StreamTestButton: React.FC = ({ readStream }) => { - const { editorView, yamlIsValid, testStreamIndex } = useConnectorBuilderState(); +export const StreamTestButton: React.FC = ({ + readStream, + hasConfigJsonErrors, + setTestInputOpen, +}) => { + const { editorView, yamlIsValid } = useConnectorBuilderState(); const { hasErrors, validateAndTouch } = useBuilderErrors(); const handleClick = () => { + if (hasConfigJsonErrors) { + setTestInputOpen(true); + return; + } if (editorView === "yaml") { readStream(); return; } - validateAndTouch(readStream, ["global", testStreamIndex]); + validateAndTouch(readStream); }; let buttonDisabled = false; @@ -39,7 +49,7 @@ export const StreamTestButton: React.FC = ({ readStream } tooltipContent = ; } - if (editorView === "ui" && hasErrors(true, ["global", testStreamIndex])) { + if ((editorView === "ui" && hasErrors(false)) || hasConfigJsonErrors) { showWarningIcon = true; tooltipContent = ; } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index 9c0908cd3ac1..dc8027a6d0a9 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -13,7 +13,10 @@ import { ResultDisplay } from "./ResultDisplay"; import { StreamTestButton } from "./StreamTestButton"; import styles from "./StreamTester.module.scss"; -export const StreamTester: React.FC = () => { +export const StreamTester: React.FC<{ + hasConfigJsonErrors: boolean; + setTestInputOpen: (open: boolean) => void; +}> = ({ hasConfigJsonErrors, setTestInputOpen }) => { const { formatMessage } = useIntl(); const { jsonManifest, configJson, streams, testStreamIndex } = useConnectorBuilderState(); const { @@ -55,7 +58,11 @@ export const StreamTester: React.FC = () => { {streams[testStreamIndex]?.url}     - + {isFetching && (
    diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss index 3b1d67fa022d..fa2474e5efbf 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss @@ -15,8 +15,6 @@ $buttonHeight: 36px; position: absolute; top: variables.$spacing-lg; left: variables.$spacing-lg; - height: $buttonHeight; - width: $buttonHeight; } .streamSelector { diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx index 6cc2632bf6ad..6ac005ae807a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx @@ -1,10 +1,15 @@ -import React from "react"; +import React, { useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; +import { ValidationError } from "yup"; import { Heading } from "components/ui/Heading"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; +import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; +import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; +import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; +import { Spec } from "core/request/ConnectorManifest"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { links } from "utils/links"; @@ -13,8 +18,30 @@ import { StreamSelector } from "./StreamSelector"; import { StreamTester } from "./StreamTester"; import styles from "./StreamTestingPanel.module.scss"; +const EMPTY_SCHEMA = {}; + +function useConfigJsonErrors(configJson: StreamReadRequestBodyConfig, spec?: Spec): number { + return useMemo(() => { + try { + const jsonSchema = spec && spec.connection_specification ? spec.connection_specification : EMPTY_SCHEMA; + const formFields = jsonSchemaToFormBlock(jsonSchema); + const validationSchema = buildYupFormForJsonSchema(jsonSchema, formFields); + validationSchema.validateSync(configJson, { abortEarly: false }); + return 0; + } catch (e) { + if (ValidationError.isError(e)) { + return e.errors.length; + } + return 1; + } + }, [configJson, spec]); +} + export const StreamTestingPanel: React.FC = () => { - const { jsonManifest, streamListErrorMessage, yamlEditorIsMounted } = useConnectorBuilderState(); + const [isTestInputOpen, setTestInputOpen] = useState(false); + const { jsonManifest, configJson, streamListErrorMessage, yamlEditorIsMounted } = useConnectorBuilderState(); + + const configJsonErrors = useConfigJsonErrors(configJson, jsonManifest.spec); if (!yamlEditorIsMounted) { return ( @@ -38,10 +65,15 @@ export const StreamTestingPanel: React.FC = () => { )} {hasStreams && streamListErrorMessage === undefined && ( <> - +
    - + 0} setTestInputOpen={setTestInputOpen} />
    )} diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index ff673ec7175d..02cbb50f3702 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -1,82 +1,477 @@ +import { JSONSchema7 } from "json-schema"; import * as yup from "yup"; -import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorManifest"; +import { AirbyteJSONSchema } from "core/jsonSchema/types"; +import { + ConnectorManifest, + InterpolatedRequestOptionsProvider, + Spec, + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + DeclarativeStream, + NoAuth, + SessionTokenAuthenticator, + RequestOption, + OAuthAuthenticator, + DefaultPaginatorPaginationStrategy, + SimpleRetrieverStreamSlicer, + HttpRequesterAuthenticator, +} from "core/request/ConnectorManifest"; + +export interface BuilderFormInput { + key: string; + required: boolean; + definition: AirbyteJSONSchema; +} + +type BuilderFormAuthenticator = ( + | NoAuth + | (Omit & { + refresh_request_body: Array<[string, string]>; + }) + | ApiKeyAuthenticator + | BearerAuthenticator + | BasicHttpAuthenticator + | SessionTokenAuthenticator +) & { type: string }; export interface BuilderFormValues { global: { connectorName: string; urlBase: string; + authenticator: BuilderFormAuthenticator; }; + inputs: BuilderFormInput[]; + inferredInputOverrides: Record>; streams: BuilderStream[]; } +export interface BuilderPaginator { + strategy: DefaultPaginatorPaginationStrategy; + pageTokenOption: RequestOption; + pageSizeOption?: RequestOption; +} + export interface BuilderStream { name: string; urlPath: string; fieldPointer: string[]; + primaryKey: string[]; httpMethod: "GET" | "POST"; requestOptions: { requestParameters: Array<[string, string]>; requestHeaders: Array<[string, string]>; requestBody: Array<[string, string]>; }; + paginator?: BuilderPaginator; + streamSlicer?: SimpleRetrieverStreamSlicer; } +export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { + global: { + connectorName: "", + urlBase: "", + authenticator: { type: "NoAuth" }, + }, + inputs: [], + inferredInputOverrides: {}, + streams: [], +}; + +export const DEFAULT_BUILDER_STREAM_VALUES: BuilderStream = { + name: "", + urlPath: "", + fieldPointer: [], + primaryKey: [], + httpMethod: "GET", + requestOptions: { + requestParameters: [], + requestHeaders: [], + requestBody: [], + }, +}; + +function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] { + if (values.global.authenticator.type === "ApiKeyAuthenticator") { + return [ + { + key: "api_key", + required: true, + definition: { + type: "string", + title: "API Key", + airbyte_secret: true, + }, + }, + ]; + } + if (values.global.authenticator.type === "BearerAuthenticator") { + return [ + { + key: "api_key", + required: true, + definition: { + type: "string", + title: "API Key", + airbyte_secret: true, + }, + }, + ]; + } + if (values.global.authenticator.type === "BasicHttpAuthenticator") { + return [ + { + key: "username", + required: true, + definition: { + type: "string", + title: "Username", + }, + }, + { + key: "password", + required: true, + definition: { + type: "string", + title: "Password", + airbyte_secret: true, + }, + }, + ]; + } + if (values.global.authenticator.type === "OAuthAuthenticator") { + return [ + { + key: "client_id", + required: true, + definition: { + type: "string", + title: "Client ID", + airbyte_secret: true, + }, + }, + { + key: "client_secret", + required: true, + definition: { + type: "string", + title: "Client secret", + airbyte_secret: true, + }, + }, + { + key: "refresh_token", + required: true, + definition: { + type: "string", + title: "Refresh token", + airbyte_secret: true, + }, + }, + ]; + } + if (values.global.authenticator.type === "SessionTokenAuthenticator") { + return [ + { + key: "username", + required: false, + definition: { + type: "string", + title: "Username", + }, + }, + { + key: "password", + required: false, + definition: { + type: "string", + title: "Password", + airbyte_secret: true, + }, + }, + { + key: "session_token", + required: false, + definition: { + type: "string", + title: "Session token", + description: "Session token generated by user (if provided username and password are not required)", + airbyte_secret: true, + }, + }, + ]; + } + return []; +} + +export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] { + const inferredInputs = getInferredInputList(values); + return inferredInputs.map((input) => + values.inferredInputOverrides[input.key] + ? { + ...input, + definition: { ...input.definition, ...values.inferredInputOverrides[input.key] }, + } + : input + ); +} + +export const injectIntoValues = ["request_parameter", "header", "path", "body_data", "body_json"]; +const nonPathRequestOptionSchema = yup + .object() + .shape({ + inject_into: yup.mixed().oneOf(injectIntoValues.filter((val) => val !== "path")), + field_name: yup.string().required("form.empty.error"), + }) + .notRequired() + .default(undefined); + +// eslint-disable-next-line no-useless-escape +export const timeDeltaRegex = /^(([\.\d]+?)y)?(([\.\d]+?)m)?(([\.\d]+?)w)?(([\.\d]+?)d)?$/; + export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ connectorName: yup.string().required("form.empty.error"), urlBase: yup.string().required("form.empty.error"), + authenticator: yup.object({ + header: yup.mixed().when("type", { + is: (type: string) => type === "ApiKeyAuthenticator" || type === "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + token_refresh_endpoint: yup.mixed().when("type", { + is: "OAuthAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + session_token_response_key: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + login_url: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + validate_session_url: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + }), }), streams: yup.array().of( yup.object().shape({ name: yup.string().required("form.empty.error"), urlPath: yup.string().required("form.empty.error"), fieldPointer: yup.array().of(yup.string()), + primaryKey: yup.array().of(yup.string()), httpMethod: yup.mixed().oneOf(["GET", "POST"]), requestOptions: yup.object().shape({ requestParameters: yup.array().of(yup.array().of(yup.string())), requestHeaders: yup.array().of(yup.array().of(yup.string())), requestBody: yup.array().of(yup.array().of(yup.string())), }), + paginator: yup + .object() + .shape({ + pageSizeOption: nonPathRequestOptionSchema, + pageTokenOption: yup.object().shape({ + inject_into: yup.mixed().oneOf(injectIntoValues), + field_name: yup.mixed().when("inject_into", { + is: "path", + then: (schema) => schema.strip(), + otherwise: yup.string().required("form.empty.error"), + }), + }), + strategy: yup + .object({ + page_size: yup.mixed().when("type", { + is: (val: string) => ["OffsetIncrement", "PageIncrement"].includes(val), + then: yup.number().required("form.empty.error"), + otherwise: yup.number(), + }), + cursor_value: yup.mixed().when("type", { + is: "CursorPagination", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + stop_condition: yup.mixed().when("type", { + is: "CursorPagination", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + start_from_page: yup.mixed().when("type", { + is: "PageIncrement", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + }) + .notRequired() + .default(undefined), + }) + .notRequired() + .default(undefined), + streamSlicer: yup + .object() + .shape({ + cursor_field: yup.string().required("form.empty.error"), + slice_values: yup.mixed().when("type", { + is: "ListStreamSlicer", + then: yup.array().of(yup.string()), + otherwise: (schema) => schema.strip(), + }), + request_option: nonPathRequestOptionSchema, + start_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + end_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + step: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().matches(timeDeltaRegex, "form.pattern.error").required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + datetime_format: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + start_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: nonPathRequestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + end_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: nonPathRequestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + stream_state_field_start: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + stream_state_field_end: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + lookback_window: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + }) + .notRequired() + .default(undefined), }) ), }); +function builderFormAuthenticatorToAuthenticator( + globalSettings: BuilderFormValues["global"] +): HttpRequesterAuthenticator { + if (globalSettings.authenticator.type === "OAuthAuthenticator") { + return { + ...globalSettings.authenticator, + refresh_request_body: Object.fromEntries(globalSettings.authenticator.refresh_request_body), + }; + } + if (globalSettings.authenticator.type === "SessionTokenAuthenticator") { + return { + ...globalSettings.authenticator, + api_url: globalSettings.urlBase, + }; + } + return globalSettings.authenticator as HttpRequesterAuthenticator; +} + export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { + type: "DeclarativeStream", name: stream.name, + primary_key: stream.primaryKey, retriever: { + type: "SimpleRetriever", name: stream.name, + primary_key: stream.primaryKey, requester: { + type: "HttpRequester", name: stream.name, url_base: values.global?.urlBase, path: stream.urlPath, request_options_provider: { + // TODO can't declare type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. + // type: "InterpolatedRequestOptionsProvider", request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), - request_body_data: Object.fromEntries(stream.requestOptions.requestBody), - }, + request_body_json: Object.fromEntries(stream.requestOptions.requestBody), + } as InterpolatedRequestOptionsProvider, + authenticator: builderFormAuthenticatorToAuthenticator(values.global), // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema config: {}, }, record_selector: { + type: "RecordSelector", extractor: { + type: "DpathExtractor", field_pointer: stream.fieldPointer, - config: {}, }, }, + paginator: stream.paginator + ? { + type: "DefaultPaginator", + page_token_option: { + ...stream.paginator.pageTokenOption, + // ensures that empty field_name is not set, as connector builder server cannot accept a field_name if inject_into is set to 'path' + field_name: stream.paginator.pageTokenOption?.field_name + ? stream.paginator.pageTokenOption?.field_name + : undefined, + }, + page_size_option: stream.paginator.pageSizeOption, + pagination_strategy: stream.paginator.strategy, + url_base: values.global?.urlBase, + } + : { type: "NoPagination" }, + stream_slicer: stream.streamSlicer, config: {}, }, - config: {}, }; }); + const allInputs = [...values.inputs, ...getInferredInputs(values)]; + + const specSchema: JSONSchema7 = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + required: allInputs.filter((input) => input.required).map((input) => input.key), + properties: Object.fromEntries(allInputs.map((input) => [input.key, input.definition])), + additionalProperties: true, + }; + + const spec: Spec = { + connection_specification: specSchema, + documentation_url: "", + type: "Spec", + }; + return { version: "0.1.0", + type: "DeclarativeSource", check: { + type: "CheckStream", stream_names: [], }, streams: manifestStreams, + spec, }; }; diff --git a/airbyte-webapp/src/components/ui/Input/Input.module.scss b/airbyte-webapp/src/components/ui/Input/Input.module.scss index 1d6d0f9f7a6f..ebdcad777282 100644 --- a/airbyte-webapp/src/components/ui/Input/Input.module.scss +++ b/airbyte-webapp/src/components/ui/Input/Input.module.scss @@ -57,7 +57,8 @@ color: colors.$grey-300; } - &.disabled { + &.disabled, + &:read-only { pointer-events: none; color: colors.$grey-400; } diff --git a/airbyte-webapp/src/core/domain/connector/connector.ts b/airbyte-webapp/src/core/domain/connector/connector.ts index 6f0749c43952..d9452f34c211 100644 --- a/airbyte-webapp/src/core/domain/connector/connector.ts +++ b/airbyte-webapp/src/core/domain/connector/connector.ts @@ -1,6 +1,8 @@ +import { DestinationDefinitionSpecificationRead, SourceDefinitionSpecificationRead } from "core/request/AirbyteClient"; + import { DEV_IMAGE_TAG } from "./constants"; import { isSource, isSourceDefinition, isSourceDefinitionSpecification } from "./source"; -import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorT } from "./types"; +import { ConnectorDefinition, ConnectorT } from "./types"; export class Connector { static id(connector: ConnectorDefinition): string { @@ -26,7 +28,7 @@ export class ConnectorHelper { } export class ConnectorSpecification { - static id(connector: ConnectorDefinitionSpecification): string { + static id(connector: DestinationDefinitionSpecificationRead | SourceDefinitionSpecificationRead): string { return isSourceDefinitionSpecification(connector) ? connector.sourceDefinitionId : connector.destinationDefinitionId; diff --git a/airbyte-webapp/src/core/domain/connector/source.ts b/airbyte-webapp/src/core/domain/connector/source.ts index 9ecb47a49962..ae43b1df16e4 100644 --- a/airbyte-webapp/src/core/domain/connector/source.ts +++ b/airbyte-webapp/src/core/domain/connector/source.ts @@ -1,5 +1,15 @@ -import { SourceDefinitionRead, SourceDefinitionSpecificationRead, SourceRead } from "../../request/AirbyteClient"; -import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorT } from "./types"; +import { + DestinationDefinitionSpecificationRead, + SourceDefinitionRead, + SourceDefinitionSpecificationRead, + SourceRead, +} from "../../request/AirbyteClient"; +import { + ConnectorDefinition, + ConnectorDefinitionSpecification, + ConnectorT, + SourceDefinitionSpecificationDraft, +} from "./types"; export function isSource(connector: ConnectorT): connector is SourceRead { return "sourceId" in connector; @@ -15,5 +25,14 @@ export function isSourceDefinitionSpecification( return (connector as SourceDefinitionSpecificationRead).sourceDefinitionId !== undefined; } +export function isSourceDefinitionSpecificationDraft( + connector: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft +): connector is SourceDefinitionSpecificationDraft { + return ( + (connector as SourceDefinitionSpecificationRead).sourceDefinitionId === undefined && + (connector as DestinationDefinitionSpecificationRead).destinationDefinitionId === undefined + ); +} + // eslint-disable-next-line no-template-curly-in-string export const SOURCE_NAMESPACE_TAG = "${SOURCE_NAMESPACE}"; diff --git a/airbyte-webapp/src/core/domain/connector/types.ts b/airbyte-webapp/src/core/domain/connector/types.ts index d52ec5feb3c2..444af7d0e98b 100644 --- a/airbyte-webapp/src/core/domain/connector/types.ts +++ b/airbyte-webapp/src/core/domain/connector/types.ts @@ -10,6 +10,11 @@ import { export type ConnectorDefinition = SourceDefinitionReadWithLatestTag | DestinationDefinitionReadWithLatestTag; +export type SourceDefinitionSpecificationDraft = Pick< + SourceDefinitionSpecificationRead, + "documentationUrl" | "connectionSpecification" | "authSpecification" | "advancedAuth" +>; + export type ConnectorDefinitionSpecification = | DestinationDefinitionSpecificationRead | SourceDefinitionSpecificationRead; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index d676dc36bda3..e6e0014514db 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -639,7 +639,7 @@ "connectorBuilder.downloadYaml": "Download Config", "connectorBuilder.testButton": "Test", - "connectorBuilder.configMenuTitle": "Configure Test Input", + "connectorBuilder.configMenuTitle": "User Inputs", "connectorBuilder.configMenuConfirm": "Confirm", "connectorBuilder.recordsTab": "Records", "connectorBuilder.requestTab": "Request", @@ -666,6 +666,7 @@ "connectorBuilder.resetModal.submitButton": "Reset", "connectorBuilder.streamsHeading": "STREAMS ({number})", "connectorBuilder.globalConfiguration": "Global Configuration", + "connectorBuilder.userInputs": "User inputs ({number})", "connectorBuilder.noStreamsMessage": "Add a stream to test it here", "connectorBuilder.toggleModal.text": "Toggling back to the UI will erase any changes you have made in the YAML editor.\n\nIn order to export your current yaml, click the Download Config button.", "connectorBuilder.toggleModal.title": "Warning", @@ -678,9 +679,48 @@ "connectorBuilder.uiYamlToggle.yaml": "YAML", "connectorBuilder.resetAll": "Reset all", "connectorBuilder.emptyName": "(empty)", + "connectorBuilder.inputsTitle": "User Inputs", + "connectorBuilder.inputsDescription": "User inputs will be asked to the end-user in order to set up the connector.", + "connectorBuilder.addInputButton": "Add new user input", + "connectorBuilder.inputModal.newTitle": "New user input", + "connectorBuilder.inputModal.editTitle": "Edit user input", + "connectorBuilder.inputModal.inputName": "Input name", + "connectorBuilder.inputModal.inputNameTooltip": "The name of the input as it will show up in the form when configuring a connection", + "connectorBuilder.inputModal.fieldId": "Field ID", + "connectorBuilder.inputModal.fieldIdTooltip": "The ID of the field - reference it in double curly braces when referencing the value of the input somewhere in the definition of a stream or global configuration: {syntaxExample}. This value is derived automatically from the input name", + "connectorBuilder.inputModal.description": "Hint", + "connectorBuilder.inputModal.descriptionTooltip": "The hint shown in the tooltip next to the input in the form when confguring a connection", + "connectorBuilder.inputModal.type": "Type", + "connectorBuilder.inputModal.typeTooltip": "The type of the input", + "connectorBuilder.inputModal.secret": "Secret field", + "connectorBuilder.inputModal.secretTooltip": "If this option is enabled, the form input will be masked and can't be looked at again", + "connectorBuilder.inputModal.required": "Required field", + "connectorBuilder.inputModal.requiredTooltip": "If this option is enabled, the user will need to provide a value when confguring a connection", + "connectorBuilder.inputModal.showDefaultValueField": "Enable default value", + "connectorBuilder.inputModal.showDefaultValueFieldTooltip": "If a default value is provided, the input is prefilled in the configuration form", + "connectorBuilder.inputModal.default": "Default value", + "connectorBuilder.inputModal.placeholder": "Placeholder", + "connectorBuilder.inputModal.placeholderTooltip": "Placeholder shown in the form in case the user did not pick a value yet", + "connectorBuilder.inputModal.enum": "Allowed values", + "connectorBuilder.inputModal.enumTooltip": "The user will only be able to choose from one of these values. If none are provided the user will be able to enter any value", + "connectorBuilder.inputModal.unsupportedInput": "Detailed configuration for this property type is disabled, switch to YAML view to edit", + "connectorBuilder.inputModal.inferredInputMessage": "Detailed configuration for this user input is disabled as it is tied to the selected authentication method", "connectorBuilder.key": "key", "connectorBuilder.value": "value", "connectorBuilder.addKeyValue": "Add", + "connectorBuilder.saveInputsForm": "Save", + "connectorBuilder.inputsFormWarning": "User inputs are not saved with the connector. They are required in order to test your streams, and will be asked to the end user in order to setup this connector", + "connectorBuilder.inputsError": "User inputs form could not be rendered: {error}. Make sure the spec in the YAML conforms to the specified standard.", + "connectorBuilder.inputsErrorDocumentation": "Check out the documentation", + "connectorBuilder.goToYaml": "Switch to YAML view", + "connectorBuilder.close": "Close", + "connectorBuilder.inputsTooltip": "Define test inputs to check whether the connector configuration works", + "connectorBuilder.inputsNoSpecUITooltip": "Add User Input fields to allow setting test inputs", + "connectorBuilder.inputsNoSpecYAMLTooltip": "Add a spec to your manifest to allow setting test inputs", + "connectorBuilder.setInUserInput": "This setting is configured as part of the user inputs in the testing panel", + "connectorBuilder.inputsButton": "Inputs", + "connectorBuilder.optionalFieldsLabel": "Optional fields", + "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times", "jobs.noAttemptsFailure": "Failed to start job.", diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx index fc28253ee5c7..cf212e4b0c75 100644 --- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx +++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx @@ -20,36 +20,50 @@ const ConnectorBuilderPageInner: React.FC = () => { const { builderFormValues, editorView, setEditorView } = useConnectorBuilderState(); return ( - undefined} validationSchema={builderFormValidationSchema}> - {({ values }) => ( - - {editorView === "yaml" ? ( - setEditorView("ui")} /> - ) : ( - setEditorView("yaml")} /> - )} - - ), - className: styles.leftPanel, - minWidth: 100, - }} - secondPanel={{ - children: , - className: styles.rightPanel, - flex: 0.33, - minWidth: 60, - overlay: { - displayThreshold: 325, - header: formatMessage({ id: "connectorBuilder.testConnector" }), - rotation: "counter-clockwise", - }, - }} - /> - )} + undefined} + validationSchema={builderFormValidationSchema} + validateOnChange={false} + > + {({ values, validateForm }) => { + return ( + + {editorView === "yaml" ? ( + setEditorView("ui")} /> + ) : ( + setEditorView("yaml")} + validateForm={validateForm} + /> + )} + + ), + className: styles.leftPanel, + minWidth: 100, + }} + secondPanel={{ + children: , + className: styles.rightPanel, + flex: 0.33, + minWidth: 60, + overlay: { + displayThreshold: 325, + header: formatMessage({ id: "connectorBuilder.testConnector" }), + rotation: "counter-clockwise", + }, + }} + /> + ); + }} ); }; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 69b01d4e4013..356afdf64dee 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -1,33 +1,28 @@ import { dump } from "js-yaml"; -import React, { useContext, useEffect, useMemo, useState } from "react"; +import merge from "lodash/merge"; +import React, { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; -import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types"; +import { BuilderFormValues, convertToManifest, DEFAULT_BUILDER_FORM_VALUES } from "components/connectorBuilder/types"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useListStreams } from "./ConnectorBuilderApiService"; -export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { - global: { - connectorName: "", - urlBase: "", - }, - streams: [], -}; - const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { version: "0.1.0", + type: "DeclarativeSource", check: { + type: "CheckStream", stream_names: [], }, streams: [], }; -type EditorView = "ui" | "yaml"; -export type BuilderView = "global" | number; +export type EditorView = "ui" | "yaml"; +export type BuilderView = "global" | "inputs" | number; interface Context { builderFormValues: BuilderFormValues; @@ -39,16 +34,15 @@ interface Context { streamListErrorMessage: string | undefined; testStreamIndex: number; selectedView: BuilderView; - configString: string; configJson: StreamReadRequestBodyConfig; editorView: EditorView; - setBuilderFormValues: (values: BuilderFormValues) => void; + setBuilderFormValues: (values: BuilderFormValues, isInvalid: boolean) => void; setJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; setTestStreamIndex: (streamIndex: number) => void; setSelectedView: (view: BuilderView) => void; - setConfigString: (configString: string) => void; + setConfigJson: (value: StreamReadRequestBodyConfig) => void; setEditorView: (editorView: EditorView) => void; } @@ -58,11 +52,26 @@ export const ConnectorBuilderStateProvider: React.FC( + const [storedBuilderFormValues, setStoredBuilderFormValues] = useLocalStorage( "connectorBuilderFormValues", DEFAULT_BUILDER_FORM_VALUES ); - const formValues = builderFormValues ?? DEFAULT_BUILDER_FORM_VALUES; + + const lastValidBuilderFormValuesRef = useRef(storedBuilderFormValues as BuilderFormValues); + + const setBuilderFormValues = useCallback( + (values: BuilderFormValues, isValid: boolean) => { + setStoredBuilderFormValues(values); + if (isValid) { + lastValidBuilderFormValuesRef.current = values; + } + }, + [setStoredBuilderFormValues] + ); + + const builderFormValues = useMemo(() => { + return merge({}, DEFAULT_BUILDER_FORM_VALUES, storedBuilderFormValues); + }, [storedBuilderFormValues]); const [jsonManifest, setJsonManifest] = useLocalStorage( "connectorBuilderJsonManifest", @@ -71,8 +80,8 @@ export const ConnectorBuilderStateProvider: React.FC { - setJsonManifest(convertToManifest(formValues)); - }, [formValues, setJsonManifest]); + setJsonManifest(convertToManifest(builderFormValues)); + }, [builderFormValues, setJsonManifest]); const [yamlIsValid, setYamlIsValid] = useState(true); const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(true); @@ -84,25 +93,30 @@ export const ConnectorBuilderStateProvider: React.FC("ui"); + const lastValidBuilderFormValues = lastValidBuilderFormValuesRef.current; + /** + * The json manifest derived from the last valid state of the builder form values. + * In the yaml view, this is undefined. Can still be invalid in case an invalid state is loaded from localstorage + */ + const lastValidJsonManifest = useMemo( + () => + editorView !== "ui" + ? undefined + : builderFormValues === lastValidBuilderFormValues + ? jsonManifest + : convertToManifest(lastValidBuilderFormValues), + [builderFormValues, editorView, jsonManifest, lastValidBuilderFormValues] + ); + // config - const [configString, setConfigString] = useState("{\n \n}"); const [configJson, setConfigJson] = useState({}); - useEffect(() => { - try { - const json = JSON.parse(configString) as StreamReadRequestBodyConfig; - setConfigJson(json); - } catch (err) { - console.error(`Config value is not valid JSON! Error: ${err}`); - } - }, [configString]); - // streams const { data: streamListRead, isError: isStreamListError, error: streamListError, - } = useListStreams({ manifest, config: configJson }); + } = useListStreams({ manifest: lastValidJsonManifest || manifest, config: configJson }); const unknownErrorMessage = formatMessage({ id: "connectorBuilder.unknownError" }); const streamListErrorMessage = isStreamListError ? streamListError instanceof Error @@ -123,7 +137,7 @@ export const ConnectorBuilderStateProvider: React.FC("global"); const ctx = { - builderFormValues: formValues, + builderFormValues, jsonManifest: manifest, yamlManifest, yamlEditorIsMounted, @@ -132,7 +146,6 @@ export const ConnectorBuilderStateProvider: React.FC Promise; isEditMode?: boolean; formValues?: Partial; @@ -25,6 +32,17 @@ export interface ConnectorFormProps { errorMessage?: React.ReactNode; successMessage?: React.ReactNode; connectorId?: string; + footerClassName?: string; + bodyClassName?: string; + submitLabel?: string; + /** + * Called in case the user cancels the form - if not provided, no cancel button is rendered + */ + onCancel?: () => void; + /** + * Called in case the user reset the form - if not provided, no reset button is rendered + */ + onReset?: () => void; isTestConnectionInProgress?: boolean; onStopTesting?: () => void; @@ -46,6 +64,7 @@ export const ConnectorForm: React.FC = (props) => { selectedConnectorDefinitionSpecification, errorMessage, connectorId, + onReset, } = props; const { formFields, initialValues, validationSchema } = useBuildForm( @@ -67,7 +86,6 @@ export const ConnectorForm: React.FC = (props) => { async (values: ConnectorFormValues) => { const valuesToSend = getValues(values); await onSubmit(valuesToSend); - clearFormChange(formId); }, [clearFormChange, formId, getValues, onSubmit] @@ -82,7 +100,7 @@ export const ConnectorForm: React.FC = (props) => { onSubmit={onFormSubmit} enableReinitialize > - {({ dirty }) => ( + {({ dirty, resetForm }) => ( = (props) => { {...props} formFields={formFields} errorMessage={errorMessage} + onReset={ + onReset && + (() => { + onReset?.(); + resetForm(); + }) + } onStopTestingConnector={onStopTesting ? () => onStopTesting() : undefined} onRetest={testConnector ? async () => await testConnector() : undefined} /> diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx index d11cb3684cd0..fbc00004eafe 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx @@ -17,6 +17,17 @@ interface FormRootProps { successMessage?: React.ReactNode; onRetest?: () => void; onStopTestingConnector?: () => void; + submitLabel?: string; + footerClassName?: string; + bodyClassName?: string; + /** + * Called in case the user cancels the form - if not provided, no cancel button is rendered + */ + onCancel?: () => void; + /** + * Called in case the user reset the form - if not provided, no reset button is rendered + */ + onReset?: () => void; } export const FormRoot: React.FC = ({ @@ -27,38 +38,50 @@ export const FormRoot: React.FC = ({ errorMessage, connectionTestSuccess, onStopTestingConnector, + submitLabel, + footerClassName, + bodyClassName, + onCancel, + onReset, }) => { const { dirty, isSubmitting, isValid } = useFormikContext(); const { resetConnectorForm, isEditMode, formType } = useConnectorForm(); return (
    - - {isEditMode ? ( - { - resetConnectorForm(); - }} - successMessage={successMessage} - /> - ) : ( - - )} +
    + +
    +
    + {isEditMode ? ( + { + resetConnectorForm(); + }} + successMessage={successMessage} + /> + ) : ( + + )} +
    ); }; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss index e6049bbfa295..d93a86168241 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss @@ -1,3 +1,19 @@ -.submitButton { - margin-left: auto; +@use "scss/variables"; + +.controlContainer { + margin-top: 34px; + display: flex; + align-items: center; + justify-content: flex-end; +} + +.buttonContainer { + display: flex; + flex: 0 0 auto; + align-self: flex-end; + gap: variables.$spacing-sm; +} + +.deleteButtonContainer { + flex-grow: 1; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx index 689bf4fdca4f..425157a83cc7 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx @@ -1,6 +1,5 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; import { Button } from "components/ui/Button"; @@ -11,6 +10,15 @@ import TestingConnectionSuccess from "./TestingConnectionSuccess"; interface CreateControlProps { formType: "source" | "destination"; + /** + * Called in case the user cancels the form - if not provided, no cancel button is rendered + */ + onCancel?: () => void; + /** + * Called in case the user reset the form - if not provided, no reset button is rendered + */ + onReset?: () => void; + submitLabel?: string; isSubmitting: boolean; errorMessage?: React.ReactNode; connectionTestSuccess?: boolean; @@ -19,13 +27,6 @@ interface CreateControlProps { onCancelTesting?: () => void; } -const ButtonContainer = styled.div` - margin-top: 34px; - display: flex; - align-items: center; - justify-content: space-between; -`; - const CreateControls: React.FC = ({ isTestConnectionInProgress, isSubmitting, @@ -33,6 +34,9 @@ const CreateControls: React.FC = ({ connectionTestSuccess, errorMessage, onCancelTesting, + onCancel, + onReset, + submitLabel, }) => { if (isSubmitting) { return ; @@ -43,12 +47,26 @@ const CreateControls: React.FC = ({ } return ( - +
    {errorMessage && } - - + {onReset && ( +
    + +
    + )} +
    + {onCancel && ( + + )} + +
    +
    ); }; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.module.scss b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.module.scss new file mode 100644 index 000000000000..c9c2ad453ba2 --- /dev/null +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.module.scss @@ -0,0 +1,3 @@ +.dropdown { + min-width: calc(50% - 100px); +} diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx index cb38783101ca..7512243b1500 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx @@ -9,6 +9,7 @@ import { FormBlock, FormConditionItem } from "core/form/types"; import { isDefined } from "utils/common"; import { ConnectorFormValues } from "../../types"; +import styles from "./ConditionSection.module.scss"; import { FormSection } from "./FormSection"; import { GroupLabel } from "./GroupLabel"; import { SectionContainer } from "./SectionContainer"; @@ -70,7 +71,7 @@ export const ConditionSection: React.FC = ({ formField, p } - dropdown={ + control={ = ({ formField, p error={typeof meta.error === "string" && !!meta.error} /> } + controlClassName={styles.dropdown} > {/* currentlySelectedCondition is only falsy if a malformed config is loaded which doesn't have a valid value for the const selection key. In this case, render the selection group as empty. */} {typeof currentlySelectedCondition !== "undefined" && ( diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx index 7ad988453e15..0d35ea01ee9d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx @@ -55,9 +55,9 @@ describe("auth button", () => { it("initially renders with correct message and no status message", () => { // no auth errors mockUseConnectorForm.mockImplementationOnce(() => { - const { selectedConnectorDefinitionSpecification, selectedConnectorDefinition } = baseUseConnectorFormValues; + const { selectedConnectorDefinition } = baseUseConnectorFormValues; - return { selectedConnectorDefinitionSpecification, selectedConnectorDefinition }; + return { selectedConnectorDefinition }; }); // not done @@ -70,7 +70,11 @@ describe("auth button", () => { render( - + ); @@ -90,9 +94,9 @@ describe("auth button", () => { it("after successful authentication, it renders with correct message and success message", () => { // no auth errors mockUseConnectorForm.mockImplementationOnce(() => { - const { selectedConnectorDefinitionSpecification, selectedConnectorDefinition } = baseUseConnectorFormValues; + const { selectedConnectorDefinition } = baseUseConnectorFormValues; - return { selectedConnectorDefinitionSpecification, selectedConnectorDefinition }; + return { selectedConnectorDefinition }; }); // done @@ -105,7 +109,11 @@ describe("auth button", () => { render( - + ); @@ -123,9 +131,9 @@ describe("auth button", () => { mockUseAuthentication.mockReturnValue({ hiddenAuthFieldErrors: { field: "form.empty.error" } }); mockUseConnectorForm.mockImplementationOnce(() => { - const { selectedConnectorDefinitionSpecification, selectedConnectorDefinition } = baseUseConnectorFormValues; + const { selectedConnectorDefinition } = baseUseConnectorFormValues; - return { selectedConnectorDefinitionSpecification, selectedConnectorDefinition }; + return { selectedConnectorDefinition }; }); // not done @@ -138,7 +146,11 @@ describe("auth button", () => { render( - + ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx index ed93c3bff8d8..df7b33ab31e5 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx @@ -5,7 +5,7 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { Text } from "components/ui/Text"; -import { ConnectorSpecification } from "core/domain/connector"; +import { ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; import { ConnectorIds } from "utils/connectors"; import { useConnectorForm } from "../../../connectorFormContext"; @@ -46,16 +46,18 @@ function getAuthenticateMessageId(connectorDefinitionId: string): string { return "connectorForm.authenticate"; } -export const AuthButton: React.FC = () => { - const { selectedConnectorDefinition, selectedConnectorDefinitionSpecification } = useConnectorForm(); +export const AuthButton: React.FC<{ + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; +}> = ({ selectedConnectorDefinitionSpecification }) => { + const { selectedConnectorDefinition } = useConnectorForm(); const { hiddenAuthFieldErrors } = useAuthentication(); const authRequiredError = Object.values(hiddenAuthFieldErrors).includes("form.empty.error"); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const { loading, done, run, hasRun } = useFormikOauthAdapter(selectedConnectorDefinitionSpecification); - if (!selectedConnectorDefinitionSpecification) { - console.error("Entered non-auth flow while no connector is selected"); + if (!selectedConnectorDefinition) { + console.error("Entered non-auth flow while no supported connector is selected"); return null; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx index 1c89cf2b1e62..12b38123404d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx @@ -1,15 +1,21 @@ import React from "react"; +import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { FeatureItem, IfFeatureEnabled } from "hooks/services/Feature"; +import { useConnectorForm } from "views/Connector/ConnectorForm/connectorFormContext"; import { SectionContainer } from "../SectionContainer"; import { AuthButton } from "./AuthButton"; export const AuthSection: React.FC = () => { + const { selectedConnectorDefinitionSpecification } = useConnectorForm(); + if (isSourceDefinitionSpecificationDraft(selectedConnectorDefinitionSpecification)) { + return null; + } return ( - + ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx index 11d2f766536f..a222e14c59c9 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx @@ -2,7 +2,11 @@ import { useFormikContext } from "formik"; import React, { useContext, useMemo } from "react"; import { AnySchema } from "yup"; -import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; +import { + ConnectorDefinition, + ConnectorDefinitionSpecification, + SourceDefinitionSpecificationDraft, +} from "core/domain/connector"; import { ConnectorFormValues } from "./types"; @@ -10,8 +14,8 @@ interface ConnectorFormContext { formType: "source" | "destination"; getValues: (values: ConnectorFormValues) => ConnectorFormValues; resetConnectorForm: () => void; - selectedConnectorDefinition: ConnectorDefinition; - selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; + selectedConnectorDefinition?: ConnectorDefinition; + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft; isEditMode?: boolean; validationSchema: AnySchema; connectorId?: string; @@ -28,11 +32,11 @@ export const useConnectorForm = (): ConnectorFormContext => { }; interface ConnectorFormContextProviderProps { - selectedConnectorDefinition: ConnectorDefinition; + selectedConnectorDefinition?: ConnectorDefinition; formType: "source" | "destination"; isEditMode?: boolean; getValues: (values: ConnectorFormValues) => ConnectorFormValues; - selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft; validationSchema: AnySchema; connectorId?: string; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx index 9ca0171a17b6..d391f35ee0a2 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx @@ -3,7 +3,7 @@ import withMock from "storybook-addon-mock"; import { Card } from "components/ui/Card"; -import { ConnectorSpecification } from "core/domain/connector"; +import { ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; import { isSourceDefinitionSpecification } from "core/domain/connector/source"; import { ConnectorForm } from "./ConnectorForm"; @@ -46,13 +46,11 @@ export default { } as ComponentMeta; const Template: ComponentStory = (args) => { + const selectedSpecification = args.selectedConnectorDefinitionSpecification as ConnectorDefinitionSpecification; // Hack to allow devs to not specify sourceDefinitionId - if ( - args.selectedConnectorDefinitionSpecification && - !ConnectorSpecification.id(args.selectedConnectorDefinitionSpecification) - ) { - if (isSourceDefinitionSpecification(args.selectedConnectorDefinitionSpecification)) { - args.selectedConnectorDefinitionSpecification.sourceDefinitionId = TempConnector.sourceDefinitionId; + if (!ConnectorSpecification.id(selectedSpecification)) { + if (isSourceDefinitionSpecification(selectedSpecification)) { + selectedSpecification.sourceDefinitionId = TempConnector.sourceDefinitionId; } } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx index 7ec5a265e1b6..9610895dcf42 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx @@ -3,6 +3,7 @@ import { JSONSchema7 } from "json-schema"; import { useCallback, useMemo } from "react"; import { ConnectorSpecification } from "core/domain/connector"; +import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { FeatureItem, useFeature } from "hooks/services/Feature"; @@ -155,7 +156,10 @@ export const useAuthentication = (): AuthenticationHook => { console.error(`getValues in useAuthentication failed.`, e); trackError(e, { id: "useAuthentication.getValues", - connector: connectorSpec ? ConnectorSpecification.id(connectorSpec) : null, + connector: + connectorSpec && !isSourceDefinitionSpecificationDraft(connectorSpec) + ? ConnectorSpecification.id(connectorSpec) + : null, }); return values; } @@ -178,7 +182,7 @@ export const useAuthentication = (): AuthenticationHook => { const implicitAuthFieldPaths = useMemo( () => [ // Fields from `advancedAuth` connectors - ...(advancedAuth + ...(advancedAuth && !isSourceDefinitionSpecificationDraft(connectorSpec) ? Object.values(serverProvidedOauthPaths(connectorSpec)).map((f) => makeConnectionConfigurationPath(f.path_in_connector_config) ) diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx index fdaaa82cbbb6..041ec792d02b 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx @@ -3,7 +3,12 @@ import { useMemo } from "react"; import { useIntl } from "react-intl"; import { AnySchema } from "yup"; -import { ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; +import { + ConnectorDefinitionSpecification, + ConnectorSpecification, + SourceDefinitionSpecificationDraft, +} from "core/domain/connector"; +import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { FormBuildError, isFormBuildError } from "core/form/FormBuildError"; import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; @@ -17,23 +22,29 @@ export interface BuildFormHook { validationSchema: AnySchema; } -function setDefaultValues(formGroup: FormGroupItem, values: Record) { +function setDefaultValues( + formGroup: FormGroupItem, + values: Record, + options: { respectExistingValues: boolean } = { respectExistingValues: false } +) { formGroup.properties.forEach((property) => { - if (property.const) { + if (property.const && (!options.respectExistingValues || !values[property.fieldKey])) { values[property.fieldKey] = property.const; } - if (property.default) { + if (property.default && (!options.respectExistingValues || !values[property.fieldKey])) { values[property.fieldKey] = property.default; } switch (property._type) { case "formGroup": - values[property.fieldKey] = {}; - setDefaultValues(property, values[property.fieldKey] as Record); + values[property.fieldKey] = + options.respectExistingValues && values[property.fieldKey] ? values[property.fieldKey] : {}; + setDefaultValues(property, values[property.fieldKey] as Record, options); break; case "formCondition": // implicitly select the first option (do not respect a potential default value) - values[property.fieldKey] = {}; - setDefaultValues(property.conditions[0], values[property.fieldKey] as Record); + values[property.fieldKey] = + options.respectExistingValues && values[property.fieldKey] ? values[property.fieldKey] : {}; + setDefaultValues(property.conditions[0], values[property.fieldKey] as Record, options); } }); } @@ -41,28 +52,34 @@ function setDefaultValues(formGroup: FormGroupItem, values: Record ): BuildFormHook { const { formatMessage } = useIntl(); + const isDraft = isSourceDefinitionSpecificationDraft(selectedConnectorDefinitionSpecification); try { - const jsonSchema: JSONSchema7 = useMemo( - () => ({ + const jsonSchema: JSONSchema7 = useMemo(() => { + const schema: JSONSchema7 = { type: "object", properties: { - name: { - type: "string", - title: formatMessage({ id: `form.${formType}Name` }), - description: formatMessage({ id: `form.${formType}Name.message` }), - }, connectionConfiguration: selectedConnectorDefinitionSpecification.connectionSpecification as JSONSchema7Definition, }, - required: ["name"], - }), - [formType, formatMessage, selectedConnectorDefinitionSpecification.connectionSpecification] - ); + }; + if (isDraft) { + return schema; + } + // schema.properties gets defined right above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + schema.properties!.name = { + type: "string", + title: formatMessage({ id: `form.${formType}Name` }), + description: formatMessage({ id: `form.${formType}Name.message` }), + }; + schema.required = ["name"]; + return schema; + }, [formType, formatMessage, isDraft, selectedConnectorDefinitionSpecification.connectionSpecification]); const formFields = useMemo(() => jsonSchemaToFormBlock(jsonSchema), [jsonSchema]); @@ -70,26 +87,33 @@ export function useBuildForm( throw new FormBuildError("connectorForm.error.topLevelNonObject"); } + const validationSchema = useMemo(() => buildYupFormForJsonSchema(jsonSchema, formFields), [formFields, jsonSchema]); + const startValues = useMemo(() => { - if (isEditMode) { - return { - name: "", - connectionConfiguration: {}, - ...initialValues, - }; - } - const baseValues = { + let baseValues = { name: "", connectionConfiguration: {}, ...initialValues, }; - setDefaultValues(formFields, baseValues as Record); + if (isDraft) { + try { + baseValues = validationSchema.cast(baseValues, { stripUnknown: true }); + } catch { + // cast did not work which can happen if there are unexpected values in the form. Reset form in this case + baseValues.connectionConfiguration = {}; + } + } + + if (isEditMode) { + return baseValues; + } + + setDefaultValues(formFields, baseValues as Record, { respectExistingValues: isDraft }); return baseValues; - }, [formFields, initialValues, isEditMode]); + }, [formFields, initialValues, isDraft, isEditMode, validationSchema]); - const validationSchema = useMemo(() => buildYupFormForJsonSchema(jsonSchema, formFields), [formFields, jsonSchema]); return { initialValues: startValues, formFields, @@ -98,7 +122,10 @@ export function useBuildForm( } catch (e) { // catch and re-throw form-build errors to enrich them with the connector id if (isFormBuildError(e)) { - throw new FormBuildError(e.message, ConnectorSpecification.id(selectedConnectorDefinitionSpecification)); + throw new FormBuildError( + e.message, + isDraft ? undefined : ConnectorSpecification.id(selectedConnectorDefinitionSpecification) + ); } throw e; }