🪟🎉 Connector builder: Add manual schema

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderConfigView.module.scss

@@ -5,6 +5,7 @@
   display: flex;
   flex-direction: column;
   gap: variables.$spacing-md;
+  height: 100%;
 }
 
 .heading {

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOptional.tsx

@@ -11,6 +11,7 @@ export const BuilderOptional: React.FC<React.PropsWithChildren<unknown>> = ({ ch
   return (
     <div className={styles.wrapper}>
       <button
+        type="button"
         onClick={() => {
           setIsOpen(!isOpen);
         }}

airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss

@@ -6,7 +6,8 @@ $controlButtonWidth: 24px;
 .controls {
   display: flex;
   justify-content: center;
-  gap: variables.$spacing-sm;
+  align-items: center;
+  gap: variables.$spacing-md;
 }
 
 .controlButton {
@@ -33,6 +34,33 @@ $controlButtonWidth: 24px;
   }
 }
 
+.tab {
+  background: none;
+  border: none;
+  padding: variables.$spacing-md;
+  border-bottom: variables.$border-thick solid transparent;
+  color: colors.$grey-400;
+  cursor: pointer;
+  display: flex;
+  gap: variables.$spacing-md;
+  align-items: center;
+}
+
+.selectedTab {
+  color: colors.$black;
+  border-bottom-color: colors.$yellow-900;
+}
+
+.errorMessage {
+  // hardcode height to prevent resizing when error message is hidden
+  height: 16px;
+  color: colors.$red;
+}
+
+.schemaEditor {
+  height: 100%;
+}
+
 .multiStreams {
   padding-right: 50px;
 }

airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx

@@ -2,7 +2,12 @@ import { faTrashCan, faCopy } from "@fortawesome/free-regular-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import classNames from "classnames";
 import { useField } from "formik";
-import { useIntl } from "react-intl";
+import { useState } from "react";
+import { FormattedMessage, useIntl } from "react-intl";
+
+import Indicator from "components/Indicator";
+import { CodeEditor } from "components/ui/CodeEditor";
+import { Text } from "components/ui/Text";
 
 import { useConfirmationModalService } from "hooks/services/ConfirmationModal";
 import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService";
@@ -26,6 +31,7 @@ interface StreamConfigViewProps {
 export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, hasMultipleStreams }) => {
   const { formatMessage } = useIntl();
   const [field, , helpers] = useField<BuilderStream[]>("streams");
+  const [selectedTab, setSelectedTab] = useState<"configuration" | "schema">("configuration");
   const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
   const { setSelectedView, setTestStreamIndex } = useConnectorBuilderState();
 
@@ -49,6 +55,9 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, h
     });
   };
 
+  const [, meta] = useField<string | undefined>(streamFieldPath("schema"));
+  const hasSchemaErrors = Boolean(meta.error);
+
   return (
     <BuilderConfigView
       heading={formatMessage({ id: "connectorBuilder.stream" })}
@@ -57,6 +66,17 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, h
       {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */}
       <BuilderTitle path={streamFieldPath("name")} label="Stream Name" size="md" />
       <div className={styles.controls}>
+        <StreamTab
+          label={formatMessage({ id: "connectorBuilder.streamConfiguration" })}
+          selected={selectedTab === "configuration"}
+          onSelect={() => setSelectedTab("configuration")}
+        />
+        <StreamTab
+          label={formatMessage({ id: "connectorBuilder.streamSchema" })}
+          selected={selectedTab === "schema"}
+          onSelect={() => setSelectedTab("schema")}
+          showErrorIndicator={hasSchemaErrors}
+        />
         <AddStreamButton
           onAddStream={(addedStreamNum) => {
             setSelectedView(addedStreamNum);
@@ -73,53 +93,96 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, h
           <FontAwesomeIcon icon={faTrashCan} />
         </button>
       </div>
-      <BuilderCard>
-        <BuilderField
-          type="string"
-          path={streamFieldPath("urlPath")}
-          label="Path URL"
-          tooltip="Path of the endpoint that this stream represents."
-        />
-        <BuilderField
-          type="enum"
-          path={streamFieldPath("httpMethod")}
-          options={["GET", "POST"]}
-          label="HTTP Method"
-          tooltip="HTTP method to use for requests sent to the API"
-        />
-        <BuilderField
-          type="array"
-          path={streamFieldPath("fieldPointer")}
-          label="Record selector"
-          tooltip="Pointer into the response that should be extracted as the final record"
-        />
-        <BuilderField
-          type="array"
-          path={streamFieldPath("primaryKey")}
-          label="Primary key"
-          tooltip="Pointer into the response that should be used as the primary key when deduplicating records in the destination"
-          optional
-        />
-      </BuilderCard>
-      <PaginationSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
-      <StreamSlicerSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
-      <BuilderCard>
-        <KeyValueListField
-          path={streamFieldPath("requestOptions.requestParameters")}
-          label="Request Parameters"
-          tooltip="Parameters to attach to API requests"
-        />
-        <KeyValueListField
-          path={streamFieldPath("requestOptions.requestHeaders")}
-          label="Request Headers"
-          tooltip="Headers to attach to API requests"
-        />
-        <KeyValueListField
-          path={streamFieldPath("requestOptions.requestBody")}
-          label="Request Body"
-          tooltip="Body to attach to API requests as url-encoded form values"
-        />
-      </BuilderCard>
+      {selectedTab === "configuration" ? (
+        <>
+          <BuilderCard>
+            <BuilderField
+              type="string"
+              path={streamFieldPath("urlPath")}
+              label="Path URL"
+              tooltip="Path of the endpoint that this stream represents."
+            />
+            <BuilderField
+              type="enum"
+              path={streamFieldPath("httpMethod")}
+              options={["GET", "POST"]}
+              label="HTTP Method"
+              tooltip="HTTP method to use for requests sent to the API"
+            />
+            <BuilderField
+              type="array"
+              path={streamFieldPath("fieldPointer")}
+              label="Record selector"
+              tooltip="Pointer into the response that should be extracted as the final record"
+            />
+            <BuilderField
+              type="array"
+              path={streamFieldPath("primaryKey")}
+              label="Primary key"
+              tooltip="Pointer into the response that should be used as the primary key when deduplicating records in the destination"
+              optional
+            />
+          </BuilderCard>
+          <PaginationSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
+          <StreamSlicerSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
+          <BuilderCard>
+            <KeyValueListField
+              path={streamFieldPath("requestOptions.requestParameters")}
+              label="Request Parameters"
+              tooltip="Parameters to attach to API requests"
+            />
+            <KeyValueListField
+              path={streamFieldPath("requestOptions.requestHeaders")}
+              label="Request Headers"
+              tooltip="Headers to attach to API requests"
+            />
+            <KeyValueListField
+              path={streamFieldPath("requestOptions.requestBody")}
+              label="Request Body"
+              tooltip="Body to attach to API requests as url-encoded form values"
+            />
+          </BuilderCard>
+        </>
+      ) : (
+        <BuilderCard className={styles.schemaEditor}>
+          <SchemaEditor streamFieldPath={streamFieldPath} />
+        </BuilderCard>
+      )}
     </BuilderConfigView>
   );
 };
+
+const StreamTab = ({
+  selected,
+  label,
+  onSelect,
+  showErrorIndicator,
+}: {
+  selected: boolean;
+  label: string;
+  onSelect: () => void;
+  showErrorIndicator?: boolean;
+}) => (
+  <button type="button" className={classNames(styles.tab, { [styles.selectedTab]: selected })} onClick={onSelect}>
+    {label}
+    {showErrorIndicator && <Indicator />}
+  </button>
+);
+
+const SchemaEditor = ({ streamFieldPath }: { streamFieldPath: (fieldPath: string) => string }) => {
+  const [field, meta, helpers] = useField<string | undefined>(streamFieldPath("schema"));
+
+  return (
+    <>
+      <CodeEditor
+        value={field.value || ""}
+        language="json"
+        theme="airbyte-light"
+        onChange={(val: string | undefined) => {
+          helpers.setValue(val);
+        }}
+      />
+      <Text className={styles.errorMessage}>{meta.error && <FormattedMessage id={meta.error} />}</Text>
+    </>
+  );
+};

airbyte-webapp/src/components/connectorBuilder/types.ts

@@ -66,6 +66,7 @@ export interface BuilderStream {
   };
   paginator?: BuilderPaginator;
   streamSlicer?: SimpleRetrieverStreamSlicer;
+  schema?: string;
 }
 
 export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = {
@@ -274,6 +275,20 @@ export const builderFormValidationSchema = yup.object().shape({
         requestHeaders: yup.array().of(yup.array().of(yup.string())),
         requestBody: yup.array().of(yup.array().of(yup.string())),
       }),
+      schema: yup.string().test({
+        test: (val: string | undefined) => {
+          if (!val) {
+            return true;
+          }
+          try {
+            JSON.parse(val);
+            return true;
+          } catch {
+            return false;
+          }
+        },
+        message: "connectorBuilder.invalidSchema",
+      }),
       paginator: yup
         .object()
         .shape({
@@ -394,12 +409,24 @@ function builderFormAuthenticatorToAuthenticator(
   return globalSettings.authenticator as HttpRequesterAuthenticator;
 }
 
+function parseSchemaString(schema?: string) {
+  if (!schema) {
+    return undefined;
+  }
+  try {
+    return { type: "InlineSchemaLoader", schema: JSON.parse(schema) };
+  } catch {
+    return undefined;
+  }
+}
+
 export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => {
   const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => {
     return {
       type: "DeclarativeStream",
       name: stream.name,
       primary_key: stream.primaryKey,
+      schema_loader: parseSchemaString(stream.schema),
       retriever: {
         type: "SimpleRetriever",
         name: stream.name,

airbyte-webapp/src/locales/en.json

@@ -718,6 +718,9 @@
   "connectorBuilder.setInUserInput": "This setting is configured as part of the user inputs in the testing panel",
   "connectorBuilder.optionalFieldsLabel": "Optional fields",
   "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times",
+  "connectorBuilder.streamConfiguration": "Configuration",
+  "connectorBuilder.streamSchema": "Schema",
+  "connectorBuilder.invalidSchema": "Invalid JSON - please fix schema to have it applied",
   "connectorBuilder.copyToPaginationTitle": "Copy pagination settings to...",
   "connectorBuilder.copyFromPaginationTitle": "Import pagination settings from...",
   "connectorBuilder.copyToSlicerTitle": "Copy slicing settings to...",