From d780fe92f709c576e2d6b9002a897989c4d9494e Mon Sep 17 00:00:00 2001 From: Joey Marshment-Howell Date: Tue, 10 Jan 2023 14:37:38 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Disable=20deselec?= =?UTF-8?q?tion=20of=20cursor=20field/primary=20key=20in=20the=20UI=20(#20?= =?UTF-8?q?844)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connection/CatalogTree/CatalogSection.tsx | 57 +++- .../connection/CatalogTree/FieldRow.tsx | 59 +++- .../CatalogTree/StreamFieldTable.tsx | 24 +- .../CatalogTree/streamConfigHelpers/index.ts | 1 + .../streamConfigHelpers.test.ts | 315 ++++++++++++++++++ .../streamConfigHelpers.ts | 120 +++++++ airbyte-webapp/src/locales/en.json | 2 + 7 files changed, 535 insertions(+), 43 deletions(-) create mode 100644 airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/index.ts create mode 100644 airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.test.ts create mode 100644 airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.ts diff --git a/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx index 7b0b0b7bcaa2..955b52e0a882 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx @@ -16,12 +16,17 @@ import { } from "core/request/AirbyteClient"; import { useDestinationNamespace } from "hooks/connection/useDestinationNamespace"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { equal, naturalComparatorBy } from "utils/objects"; +import { naturalComparatorBy } from "utils/objects"; import { ConnectionFormValues, SUPPORTED_MODES } from "views/Connection/ConnectionForm/formConfig"; import styles from "./CatalogSection.module.scss"; import { CatalogTreeTableRow } from "./next/CatalogTreeTableRow"; import { StreamDetailsPanel } from "./next/StreamDetailsPanel/StreamDetailsPanel"; +import { + updatePrimaryKey, + toggleFieldInPrimaryKey, + updateCursorField, +} from "./streamConfigHelpers/streamConfigHelpers"; import { StreamFieldTable } from "./StreamFieldTable"; import { StreamHeader } from "./StreamHeader"; import { flatten, getPathType } from "./utils"; @@ -47,6 +52,8 @@ const CatalogSectionInner: React.FC = ({ }) => { const isNewStreamsTableEnabled = process.env.REACT_APP_NEW_STREAMS_TABLE ?? false; + const numberOfFieldsInStream = Object.keys(streamNode?.stream?.jsonSchema?.properties).length ?? 0; + const { destDefinitionSpecification: { supportedDestinationSyncModes }, } = useConnectionFormService(); @@ -75,32 +82,44 @@ const CatalogSectionInner: React.FC = ({ const onPkSelect = useCallback( (pkPath: string[]) => { - let newPrimaryKey: string[][]; - - if (config?.primaryKey?.find((pk) => equal(pk, pkPath))) { - newPrimaryKey = config.primaryKey.filter((key) => !equal(key, pkPath)); - } else { - newPrimaryKey = [...(config?.primaryKey ?? []), pkPath]; + if (!config) { + return; } - updateStreamWithConfig({ primaryKey: newPrimaryKey }); + const updatedConfig = toggleFieldInPrimaryKey(config, pkPath, numberOfFieldsInStream); + + updateStreamWithConfig(updatedConfig); }, - [config?.primaryKey, updateStreamWithConfig] + [config, updateStreamWithConfig, numberOfFieldsInStream] ); const onCursorSelect = useCallback( - (cursorField: string[]) => updateStreamWithConfig({ cursorField }), - [updateStreamWithConfig] + (cursorField: string[]) => { + if (!config) { + return; + } + + const updatedConfig = updateCursorField(config, cursorField, numberOfFieldsInStream); + + updateStreamWithConfig(updatedConfig); + }, + [config, numberOfFieldsInStream, updateStreamWithConfig] ); const onPkUpdate = useCallback( - (newPrimaryKey: string[][]) => updateStreamWithConfig({ primaryKey: newPrimaryKey }), - [updateStreamWithConfig] - ); + (newPrimaryKey: string[][]) => { + if (!config) { + return; + } - const numberOfFieldsInStream = Object.keys(streamNode?.stream?.jsonSchema?.properties).length ?? 0; + const updatedConfig = updatePrimaryKey(config, newPrimaryKey, numberOfFieldsInStream); + + updateStreamWithConfig(updatedConfig); + }, + [config, updateStreamWithConfig, numberOfFieldsInStream] + ); - const onSelectedFieldsUpdate = (fieldPath: string[], isSelected: boolean) => { + const onToggleFieldSelected = (fieldPath: string[], isSelected: boolean) => { const previouslySelectedFields = config?.selectedFields || []; if (!config?.fieldSelectionEnabled && !isSelected) { @@ -226,8 +245,10 @@ const CatalogSectionInner: React.FC = ({ syncSchemaFields={flattenedFields} onCursorSelect={onCursorSelect} onPkSelect={onPkSelect} - handleFieldToggle={onSelectedFieldsUpdate} - shouldDefinePk={shouldDefinePk} + handleFieldToggle={onToggleFieldSelected} + primaryKeyIndexerType={pkType} + cursorIndexerType={cursorType} + shouldDefinePrimaryKey={shouldDefinePk} shouldDefineCursor={shouldDefineCursor} /> diff --git a/airbyte-webapp/src/components/connection/CatalogTree/FieldRow.tsx b/airbyte-webapp/src/components/connection/CatalogTree/FieldRow.tsx index 544d08d7a0bd..f1904ed1a755 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/FieldRow.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/FieldRow.tsx @@ -1,4 +1,4 @@ -import React, { memo } from "react"; +import React, { memo, useCallback } from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; @@ -15,18 +15,20 @@ import { equal } from "utils/objects"; import { useTranslateDataType } from "utils/useTranslateDataType"; import DataTypeCell from "./DataTypeCell"; -import { pathDisplayName } from "./PathPopout"; +import { IndexerType, pathDisplayName } from "./PathPopout"; import { NameContainer, SyncCheckboxContainer } from "./styles"; interface FieldRowProps { - isPrimaryKeyEnabled: boolean; - isCursorEnabled: boolean; + cursorIndexerType: IndexerType; + primaryKeyIndexerType: IndexerType; isSelected: boolean; onPrimaryKeyChange: (pk: string[]) => void; onCursorChange: (cs: string[]) => void; onToggleFieldSelected: (fieldPath: string[], isSelected: boolean) => void; field: SyncSchemaField; config: AirbyteStreamConfiguration | undefined; + shouldDefinePrimaryKey: boolean; + shouldDefineCursor: boolean; } const FirstCell = styled(Cell)` @@ -43,9 +45,11 @@ const FieldRowInner: React.FC = ({ onToggleFieldSelected, field, config, - isCursorEnabled, - isPrimaryKeyEnabled, + cursorIndexerType, + primaryKeyIndexerType, isSelected, + shouldDefinePrimaryKey, + shouldDefineCursor, }) => { const isColumnSelectionEnabled = useExperiment("connection.columnSelection", false); const dataType = useTranslateDataType(field); @@ -54,20 +58,41 @@ const FieldRowInner: React.FC = ({ const isCursor = equal(config?.cursorField, field.path); const isPrimaryKey = !!config?.primaryKey?.some((p) => equal(p, field.path)); const isNestedField = SyncSchemaFieldObject.isNestedField(field); + // The indexer type tells us whether a cursor or pk is user-defined, source-defined or not required (null) + const fieldSelectionDisabled = + (cursorIndexerType !== null && isCursor) || (primaryKeyIndexerType !== null && isPrimaryKey) || isNestedField; + + const renderDisabledReasonMessage = useCallback(() => { + if (isNestedField) { + return ; + } + if (primaryKeyIndexerType !== null && isPrimaryKey) { + return ; + } + if (cursorIndexerType !== null && isCursor) { + return ; + } + return null; + }, [isCursor, isPrimaryKey, isNestedField, field.path, cursorIndexerType, primaryKeyIndexerType]); return ( <> {isColumnSelectionEnabled && ( - {!isNestedField && ( - onToggleFieldSelected(field.path, !isSelected)} /> - )} - {isNestedField && ( - }> - - - )} + onToggleFieldSelected(field.path, !isSelected)} + /> + } + > + {renderDisabledReasonMessage()} + )} @@ -82,9 +107,11 @@ const FieldRowInner: React.FC = ({ )} {dataType} - {isCursorEnabled && onCursorChange(field.path)} />} - {isPrimaryKeyEnabled && onPrimaryKeyChange(field.path)} />} + {shouldDefineCursor && onCursorChange(field.path)} />} + + + {shouldDefinePrimaryKey && onPrimaryKeyChange(field.path)} />} {field.cleanedName} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/StreamFieldTable.tsx b/airbyte-webapp/src/components/connection/CatalogTree/StreamFieldTable.tsx index d4fd2daea569..2816c0b7906d 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/StreamFieldTable.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/StreamFieldTable.tsx @@ -1,12 +1,12 @@ import isEqual from "lodash/isEqual"; import React, { useCallback } from "react"; -import { SyncSchemaField, SyncSchemaFieldObject } from "core/domain/catalog"; +import { SyncSchemaField } from "core/domain/catalog"; import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; import { FieldHeader } from "./FieldHeader"; import { FieldRow } from "./FieldRow"; -import { pathDisplayName } from "./PathPopout"; +import { IndexerType, pathDisplayName } from "./PathPopout"; import styles from "./StreamFieldTable.module.scss"; import { TreeRowWrapper } from "./TreeRowWrapper"; @@ -15,9 +15,11 @@ interface StreamFieldTableProps { onCursorSelect: (cursorPath: string[]) => void; onPkSelect: (pkPath: string[]) => void; handleFieldToggle: (fieldPath: string[], isSelected: boolean) => void; - shouldDefineCursor: boolean; - shouldDefinePk: boolean; + cursorIndexerType: IndexerType; + primaryKeyIndexerType: IndexerType; syncSchemaFields: SyncSchemaField[]; + shouldDefinePrimaryKey: boolean; + shouldDefineCursor: boolean; } export const StreamFieldTable: React.FC = ({ @@ -25,9 +27,11 @@ export const StreamFieldTable: React.FC = ({ onCursorSelect, onPkSelect, handleFieldToggle, - shouldDefineCursor, - shouldDefinePk, + cursorIndexerType, + primaryKeyIndexerType, syncSchemaFields, + shouldDefinePrimaryKey, + shouldDefineCursor, }) => { const isFieldSelected = useCallback( (field: SyncSchemaField): boolean => { @@ -36,7 +40,7 @@ export const StreamFieldTable: React.FC = ({ return true; } - // path[0] is the top-level field name for all nested fields + // Nested fields cannot currently be individually deselected, so we can just check whether the top-level field has been selected return !!config?.selectedFields?.find((f) => isEqual(f.fieldPath, [field.path[0]])); }, [config] @@ -53,12 +57,14 @@ export const StreamFieldTable: React.FC = ({ ))} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/index.ts b/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/index.ts new file mode 100644 index 000000000000..cc0dfabc8a19 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/index.ts @@ -0,0 +1 @@ +export * from "./streamConfigHelpers"; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.test.ts b/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.test.ts new file mode 100644 index 000000000000..8e5065e42e7f --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.test.ts @@ -0,0 +1,315 @@ +import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; + +import { + mergeFieldPathArrays, + toggleFieldInPrimaryKey, + updatePrimaryKey, + updateCursorField, +} from "./streamConfigHelpers"; + +const mockStreamConfiguration: AirbyteStreamConfiguration = { + fieldSelectionEnabled: false, + selectedFields: [], + selected: true, + syncMode: "full_refresh", + destinationSyncMode: "overwrite", +}; + +const FIELD_ONE = ["field_one"]; +const FIELD_TWO = ["field_two"]; +const FIELD_THREE = ["field_three"]; + +describe(`${mergeFieldPathArrays.name}`, () => { + it("merges two arrays of fieldPaths without duplicates", () => { + const arr1 = [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }]; + const arr2 = [{ fieldPath: FIELD_TWO }, { fieldPath: FIELD_THREE }]; + + expect(mergeFieldPathArrays(arr1, arr2)).toEqual([ + { fieldPath: FIELD_ONE }, + { fieldPath: FIELD_TWO }, + { fieldPath: FIELD_THREE }, + ]); + }); + + it("merges two arrays of complex fieldPaths without duplicates", () => { + const arr1 = [{ fieldPath: [...FIELD_ONE, ...FIELD_TWO] }, { fieldPath: [...FIELD_TWO, ...FIELD_THREE] }]; + const arr2 = [ + { fieldPath: [...FIELD_ONE, ...FIELD_TWO] }, + { fieldPath: [...FIELD_TWO, ...FIELD_THREE] }, + { fieldPath: [...FIELD_ONE, ...FIELD_THREE] }, + ]; + + expect(mergeFieldPathArrays(arr1, arr2)).toEqual([ + { fieldPath: [...FIELD_ONE, ...FIELD_TWO] }, + { fieldPath: [...FIELD_TWO, ...FIELD_THREE] }, + { fieldPath: [...FIELD_ONE, ...FIELD_THREE] }, + ]); + }); +}); + +describe(`${updateCursorField.name}`, () => { + it("updates the cursor field when field selection is disabled", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + fieldSelectionEnabled: false, + selectedFields: [], + }; + + const newStreamConfiguration = updateCursorField(mockConfig, FIELD_ONE, 3); + + expect(newStreamConfiguration).toEqual({ + cursorField: FIELD_ONE, + }); + }); + describe("when fieldSelection is active", () => { + it("adds the cursor to selectedFields", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updateCursorField(mockConfig, FIELD_THREE, 100); + + expect(newStreamConfiguration).toEqual({ + cursorField: FIELD_THREE, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }, { fieldPath: FIELD_THREE }], + }); + }); + + it("updates the cursor field when only one other field is unselected", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updateCursorField(mockConfig, FIELD_ONE, 3); + + expect(newStreamConfiguration).toEqual({ + cursorField: FIELD_ONE, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }); + }); + + it("updates the cursor field when it is one of many unselected fields", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updateCursorField(mockConfig, ["new_cursor"], 100); + + expect(newStreamConfiguration).toEqual({ + cursorField: ["new_cursor"], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }, { fieldPath: ["new_cursor"] }], + }); + }); + + it("disables field selection when the selected cursor is the only unselected field", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updateCursorField(mockConfig, FIELD_THREE, 3); + + expect(newStreamConfiguration).toEqual({ + cursorField: FIELD_THREE, + fieldSelectionEnabled: false, + selectedFields: [], + }); + }); + }); +}); + +describe(`${updatePrimaryKey.name}`, () => { + it("updates the primary key field", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + }; + + const newStreamConfiguration = updatePrimaryKey(mockConfig, [FIELD_TWO], 3); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_TWO], + }); + }); + + describe("when fieldSelection is active", () => { + it("adds each piece of the composite primary key to selectedFields", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }], + }; + + const newStreamConfiguration = updatePrimaryKey(mockConfig, [FIELD_TWO, FIELD_THREE], 100); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_TWO, FIELD_THREE], + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }, { fieldPath: FIELD_THREE }], + fieldSelectionEnabled: true, + }); + }); + + it("replaces the primary key when many other field are unselected", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updatePrimaryKey(mockConfig, [FIELD_THREE], 100); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_THREE], + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }, { fieldPath: FIELD_THREE }], + fieldSelectionEnabled: true, + }); + }); + + it("replaces the primary key when only one other field is unselected", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updatePrimaryKey(mockConfig, [FIELD_TWO], 3); + + expect(newStreamConfiguration).toEqual({ + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + primaryKey: [FIELD_TWO], + }); + }); + + it("disables field selection when the selected primary key is the last unselected field", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }], + }; + + const newStreamConfiguration = updatePrimaryKey(mockConfig, [FIELD_TWO, FIELD_THREE], 3); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_TWO, FIELD_THREE], + selectedFields: [], + fieldSelectionEnabled: false, + }); + }); + + it("disables field selection when part of the selected primary key is the last unselected field", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = updatePrimaryKey(mockConfig, [FIELD_THREE], 3); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_THREE], + selectedFields: [], + fieldSelectionEnabled: false, + }); + }); + }); +}); + +describe(`${toggleFieldInPrimaryKey.name}`, () => { + it("adds a new field to the composite primary key", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + }; + + const newStreamConfiguration = toggleFieldInPrimaryKey(mockConfig, FIELD_TWO, 3); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_ONE, FIELD_TWO], + }); + }); + + describe("when fieldSelection is active", () => { + it("adds the new primary key field to selectedFields", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = toggleFieldInPrimaryKey(mockConfig, FIELD_THREE, 100); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_ONE, FIELD_THREE], + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }, { fieldPath: FIELD_THREE }], + fieldSelectionEnabled: true, + }); + }); + + it("adds the new primary key when only one other field is unselected", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = toggleFieldInPrimaryKey(mockConfig, FIELD_TWO, 3); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_TWO], + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + fieldSelectionEnabled: true, + }); + }); + + it("adds the new primary key when it is one of many unselected fields", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }], + }; + + const newStreamConfiguration = toggleFieldInPrimaryKey(mockConfig, FIELD_TWO, 100); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_TWO], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }); + }); + + it("disables field selection when selected primary key is the last unselected field", () => { + const mockConfig: AirbyteStreamConfiguration = { + ...mockStreamConfiguration, + primaryKey: [FIELD_ONE], + fieldSelectionEnabled: true, + selectedFields: [{ fieldPath: FIELD_ONE }, { fieldPath: FIELD_TWO }], + }; + + const newStreamConfiguration = toggleFieldInPrimaryKey(mockConfig, FIELD_THREE, 3); + + expect(newStreamConfiguration).toEqual({ + primaryKey: [FIELD_ONE, FIELD_THREE], + selectedFields: [], + fieldSelectionEnabled: false, + }); + }); + }); +}); diff --git a/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.ts b/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.ts new file mode 100644 index 000000000000..e3dacd8ebcef --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/streamConfigHelpers/streamConfigHelpers.ts @@ -0,0 +1,120 @@ +import isEqual from "lodash/isEqual"; + +import { AirbyteStreamConfiguration, SelectedFieldInfo } from "core/request/AirbyteClient"; + +/** + * Merges arrays of SelectedFieldInfo, ensuring there are no duplicates + */ +export function mergeFieldPathArrays(...args: SelectedFieldInfo[][]): SelectedFieldInfo[] { + const set = new Set(); + + args.forEach((array) => + array.forEach((selectedFieldInfo) => { + if (selectedFieldInfo.fieldPath) { + const key = JSON.stringify(selectedFieldInfo.fieldPath); + set.add(key); + } + }) + ); + + return Array.from(set).map((key) => ({ fieldPath: JSON.parse(key) })); +} + +/** + * Updates the cursor field in AirbyteStreamConfiguration + */ +export const updateCursorField = ( + config: AirbyteStreamConfiguration, + selectedCursorField: string[], + numberOfFieldsInStream: number +): Partial => { + // If field selection is enabled, we need to be sure the new cursor is also selected + if (config?.fieldSelectionEnabled) { + const previouslySelectedFields = config?.selectedFields || []; + const selectedFields = mergeFieldPathArrays(previouslySelectedFields, [{ fieldPath: selectedCursorField }]); + + // If the number of selected fields is equal to the fields in the stream, field selection is disabled because all fields are selected + if (selectedFields.length === numberOfFieldsInStream) { + return { cursorField: selectedCursorField, selectedFields: [], fieldSelectionEnabled: false }; + } + + return { + fieldSelectionEnabled: true, + selectedFields, + cursorField: selectedCursorField, + }; + } + return { cursorField: selectedCursorField }; +}; + +/** + * Overwrites the entire primaryKey value in AirbyteStreamConfiguration, which is a composite of one or more fieldPaths + */ +export const updatePrimaryKey = ( + config: AirbyteStreamConfiguration, + compositePrimaryKey: string[][], + numberOfFieldsInStream: number +): Partial => { + // If field selection is enabled, we need to be sure each fieldPath in the new composite primary key is also selected + if (config?.fieldSelectionEnabled) { + const previouslySelectedFields = config?.selectedFields || []; + const selectedFields = mergeFieldPathArrays( + previouslySelectedFields, + compositePrimaryKey.map((fieldPath) => ({ fieldPath })) + ); + + // If the number of selected fields is equal to the fields in the stream, field selection is disabled because all fields are selected + if (selectedFields.length === numberOfFieldsInStream) { + return { primaryKey: compositePrimaryKey, selectedFields: [], fieldSelectionEnabled: false }; + } + + return { + fieldSelectionEnabled: true, + selectedFields, + primaryKey: compositePrimaryKey, + }; + } + + return { + primaryKey: compositePrimaryKey, + }; +}; + +/** + * Toggles whether a fieldPath is part of the composite primaryKey + */ +export const toggleFieldInPrimaryKey = ( + config: AirbyteStreamConfiguration, + fieldPath: string[], + numberOfFieldsInStream: number +): Partial => { + const fieldIsSelected = !config?.primaryKey?.find((pk) => isEqual(pk, fieldPath)); + let newPrimaryKey: string[][]; + + if (!fieldIsSelected) { + newPrimaryKey = config.primaryKey?.filter((key) => !isEqual(key, fieldPath)) ?? []; + } else { + newPrimaryKey = [...(config?.primaryKey ?? []), fieldPath]; + } + + // If field selection is enabled, we need to be sure the new fieldPath is also selected + if (fieldIsSelected && config?.fieldSelectionEnabled) { + const previouslySelectedFields = config?.selectedFields || []; + const selectedFields = mergeFieldPathArrays(previouslySelectedFields, [{ fieldPath }]); + + // If the number of selected fields is equal to the fields in the stream, field selection is disabled because all fields are selected + if (selectedFields.length === numberOfFieldsInStream) { + return { primaryKey: newPrimaryKey, selectedFields: [], fieldSelectionEnabled: false }; + } + + return { + fieldSelectionEnabled: true, + selectedFields, + primaryKey: newPrimaryKey, + }; + } + + return { + primaryKey: newPrimaryKey, + }; +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index c60d46d6fa96..f1151b1dd27a 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -223,6 +223,8 @@ "form.nullable": "Nullable", "form.field.sync": "Sync", "form.field.sync.nestedFieldTooltip": "This field will be synced if \"{fieldName}\" is selected", + "form.field.sync.cursorFieldTooltip": "The cursor field cannot be deselected", + "form.field.sync.primaryKeyTooltip": "The primary key cannot be deselected", "form.field.name": "Field name", "form.field.dataType": "Data type", "form.field.destinationName": "Destination name",