Skip to content

Commit

Permalink
feat: added cron field to connections form (#16375)
Browse files Browse the repository at this point in the history
* feat: added cron field to connections form

* fix: tests are failing for formConfig

* fix: made changes requested

* fix: tests are failing for formConfig

* fix: tests are failing for formConfig

* fix: tests are failing for formConfig

* fix: tests are failing in cypress

* fix: merge conflicts

* fix: tests are failing

* fix: validation error for formik fields

* fix: tests are failing

* fix: tests are failing

* Update airbyte-webapp/src/views/Connection/ConnectionForm/components/ScheduleField.tsx

Co-authored-by: Krishna (kc) Glick <krishna@airbyte.io>

* Update airbyte-webapp/src/views/Connection/ConnectionForm/components/ScheduleField.tsx

Co-authored-by: Krishna (kc) Glick <krishna@airbyte.io>

* fix: make the requested changes

* fix: tests are failing

* fix: tests are failing

* feat: added cron test for the string

* Update airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts

Co-authored-by: Lake Mossman <lake@airbyte.io>

* fix: error message for invalid cron string

Co-authored-by: Krishna (kc) Glick <krishna@airbyte.io>
Co-authored-by: Lake Mossman <lake@airbyte.io>
  • Loading branch information
3 people authored Sep 14, 2022
1 parent 5687cef commit cfeeeed
Show file tree
Hide file tree
Showing 15 changed files with 539 additions and 83 deletions.
2 changes: 1 addition & 1 deletion airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const scheduleDropdown = "div[data-testid='scheduleData.basicSchedule']";
const scheduleDropdown = "div[data-testid='scheduleData']";
const scheduleValue = (value: string) => `div[data-testid='${value}']`;
const destinationPrefix = "input[data-testid='prefixInput']";
const replicationTab = "div[data-id='replication-step']";
Expand Down
7 changes: 4 additions & 3 deletions airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import styled from "styled-components";

import Table from "components/Table";

import { ConnectionScheduleType } from "core/request/AirbyteClient";
import { FeatureItem, useFeature } from "hooks/services/Feature";
import useRouter from "hooks/useRouter";

Expand Down Expand Up @@ -143,9 +144,9 @@ const ConnectionTable: React.FC<IProps> = ({ data, entity, onClickRow, onChangeS

{
Header: <FormattedMessage id="tables.frequency" />,
accessor: "schedule",
accessor: "scheduleData",
Cell: ({ cell, row }: CellProps<ITableDataItem>) => (
<FrequencyCell value={cell.value} enabled={row.original.enabled} />
<FrequencyCell value={cell.value} enabled={row.original.enabled} scheduleType={row.original.scheduleType} />
),
},
{
Expand Down Expand Up @@ -173,7 +174,7 @@ const ConnectionTable: React.FC<IProps> = ({ data, entity, onClickRow, onChangeS
enabled={cell.value}
id={row.original.connectionId}
isSyncing={row.original.isSyncing}
isManual={!row.original.schedule}
isManual={row.original.scheduleType === ConnectionScheduleType.manual}
onChangeStatus={onChangeStatus}
onSync={onSync}
allowSync={allowSync}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,35 @@ import React from "react";
import { FormattedMessage } from "react-intl";
import styled from "styled-components";

import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient";
import { ConnectionScheduleData, ConnectionScheduleType } from "core/request/AirbyteClient";

interface FrequencyCellProps {
value: ConnectionScheduleDataBasicSchedule;
value: ConnectionScheduleData;
enabled?: boolean;
scheduleType?: ConnectionScheduleType;
}

const Content = styled.div<{ enabled?: boolean }>`
color: ${({ theme, enabled }) => (!enabled ? theme.greyColor40 : "inherit")};
`;

const FrequencyCell: React.FC<FrequencyCellProps> = ({ value, enabled }) => (
<Content enabled={enabled}>
<FormattedMessage id={`frequency.${value ? value.timeUnit : "manual"}`} values={{ value: value?.units }} />
</Content>
);
const FrequencyCell: React.FC<FrequencyCellProps> = ({ value, enabled, scheduleType }) => {
if (scheduleType === ConnectionScheduleType.cron || scheduleType === ConnectionScheduleType.manual) {
return (
<Content enabled={enabled}>
<FormattedMessage id={`frequency.${scheduleType}`} />
</Content>
);
}

return (
<Content enabled={enabled}>
<FormattedMessage
id={`frequency.${value ? value.basicSchedule?.timeUnit : "manual"}`}
values={{ value: value.basicSchedule?.units }}
/>
</Content>
);
};

export default FrequencyCell;
5 changes: 3 additions & 2 deletions airbyte-webapp/src/components/EntityTable/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { ConnectionScheduleDataBasicSchedule } from "../../core/request/AirbyteClient";
import { ConnectionScheduleData, ConnectionScheduleType } from "../../core/request/AirbyteClient";

interface EntityTableDataItem {
entityId: string;
Expand All @@ -24,7 +24,8 @@ interface ITableDataItem {
isSyncing?: boolean;
status?: string;
lastSync?: number | null;
schedule?: ConnectionScheduleDataBasicSchedule;
scheduleData?: ConnectionScheduleData;
scheduleType?: ConnectionScheduleType;
lastSyncStatus: string | null;
connectorIcon?: string;
entityIcon?: string;
Expand Down
3 changes: 2 additions & 1 deletion airbyte-webapp/src/components/EntityTable/utils.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ export const getConnectionTableData = (
: connection[connectType]?.name || "",
lastSync: connection.latestSyncJobCreatedAt,
enabled: connection.status === ConnectionStatus.active,
schedule: connection.scheduleData?.basicSchedule,
scheduleData: connection.scheduleData,
scheduleType: connection.scheduleType,
status: connection.status,
isSyncing: connection.isSyncing,
lastSyncStatus: getConnectionSyncStatus(connection.status, connection.latestSyncJobStatus),
Expand Down
3 changes: 1 addition & 2 deletions airbyte-webapp/src/config/frequencyConfig.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient";

export const frequencyConfig: Array<ConnectionScheduleDataBasicSchedule | null> = [
null, // manual
export const frequencyConfig: ConnectionScheduleDataBasicSchedule[] = [
{
units: 1,
timeUnit: "hours",
Expand Down
5 changes: 5 additions & 0 deletions airbyte-webapp/src/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"form.yourEmail": "Your email",
"form.email.placeholder": "you@company.com",
"form.email.error": "Enter a valid email",
"form.cron.invalid": "Invalid cron expression",
"form.empty.error": "Required",
"form.selectConnector": "Type to search for a connector",
"form.searchName": "search by name...",
Expand Down Expand Up @@ -363,6 +364,9 @@
"connection.canceling": "Canceling...",

"form.frequency": "Replication frequency*",
"form.cronExpression": "Cron expression*",
"form.cronExpression.placeholder": "Cron expression",
"form.cronExpression.message": "Enter a <lnk>cron expression</lnk> for when syncs should run (ex. \"0 0 12 * * ?\" => Will sync at 12:00 PM every day)",
"form.frequency.placeholder": "Select a frequency",
"form.frequency.message": "Set how often data should sync to the destination",

Expand Down Expand Up @@ -514,6 +518,7 @@
"errorView.unknownError": "Unknown error occurred",

"frequency.manual": "Manual",
"frequency.cron": "Cron",
"frequency.minutes": "{value} min",
"frequency.hours": "{value, plural, one {# hour} other {# hours}}",

Expand Down
200 changes: 200 additions & 0 deletions airbyte-webapp/src/utils/cron.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// This comes from the fact that parseInt trims characters coming
// after digits and consider it a valid int, so `1*` becomes `1`.
const safeParseInt = (value: string): number => {
if (/^\d+$/.test(value)) {
return Number(value);
}
return NaN;
};

const isWildcard = (value: string): boolean => {
return value === "*";
};

const isQuestionMark = (value: string): boolean => {
return value === "?";
};

const isInRange = (value: number, start: number, stop: number): boolean => {
return value >= start && value <= stop;
};

const isValidRange = (value: string, start: number, stop: number): boolean => {
const sides = value.split("-");
switch (sides.length) {
case 1:
return isWildcard(value) || isInRange(safeParseInt(value), start, stop);
case 2:
const [small, big] = sides.map((side: string): number => safeParseInt(side));
return small <= big && isInRange(small, start, stop) && isInRange(big, start, stop);
default:
return false;
}
};

const isValidStep = (value: string | undefined): boolean => {
return value === undefined || (value.search(/[^\d]/) === -1 && safeParseInt(value) > 0);
};

const validateForRange = (value: string, start: number, stop: number): boolean => {
if (value.search(/[^\d-,/*]/) !== -1) {
return false;
}

const list = value.split(",");
return list.every((condition: string): boolean => {
const splits = condition.split("/");
// Prevents `*/ * * * *` from being accepted.
if (condition.trim().endsWith("/")) {
return false;
}

// Prevents `*/*/* * * * *` from being accepted
if (splits.length > 2) {
return false;
}

// If we don't have a `/`, right will be undefined which is considered a valid step if we don't a `/`.
const [left, right] = splits;
return isValidRange(left, start, stop) && isValidStep(right);
});
};

const hasValidSeconds = (seconds: string): boolean => {
return isQuestionMark(seconds) || validateForRange(seconds, 0, 59);
};

const hasValidMinutes = (minutes: string): boolean => {
return validateForRange(minutes, 0, 59);
};

const hasValidHours = (hours: string): boolean => {
return validateForRange(hours, 0, 23);
};

const hasValidDays = (days: string, allowBlankDay?: boolean): boolean => {
return (allowBlankDay && isQuestionMark(days)) || validateForRange(days, 1, 31);
};

const monthAlias: Record<string, string> = {
jan: "1",
feb: "2",
mar: "3",
apr: "4",
may: "5",
jun: "6",
jul: "7",
aug: "8",
sep: "9",
oct: "10",
nov: "11",
dec: "12",
};

const hasValidMonths = (months: string, alias?: boolean): boolean => {
// Prevents alias to be used as steps
if (months.search(/\/[a-zA-Z]/) !== -1) {
return false;
}

if (alias) {
const remappedMonths = months.toLowerCase().replace(/[a-z]{3}/g, (match: string): string => {
return monthAlias[match] === undefined ? match : monthAlias[match];
});
// If any invalid alias was used, it won't pass the other checks as there will be non-numeric values in the months
return validateForRange(remappedMonths, 1, 12);
}

return validateForRange(months, 1, 12);
};

const weekdaysAlias: Record<string, string> = {
sun: "0",
mon: "1",
tue: "2",
wed: "3",
thu: "4",
fri: "5",
sat: "6",
};

const hasValidWeekdays = (
weekdays: string,
alias?: boolean,
allowBlankDay?: boolean,
allowSevenAsSunday?: boolean
): boolean => {
// If there is a question mark, checks if the allowBlankDay flag is set
if (allowBlankDay && isQuestionMark(weekdays)) {
return true;
} else if (!allowBlankDay && isQuestionMark(weekdays)) {
return false;
}

// Prevents alias to be used as steps
if (weekdays.search(/\/[a-zA-Z]/) !== -1) {
return false;
}

if (alias) {
const remappedWeekdays = weekdays.toLowerCase().replace(/[a-z]{3}/g, (match: string): string => {
return weekdaysAlias[match] === undefined ? match : weekdaysAlias[match];
});
// If any invalid alias was used, it won't pass the other checks as there will be non-numeric values in the weekdays
return validateForRange(remappedWeekdays, 0, allowSevenAsSunday ? 7 : 6);
}

return validateForRange(weekdays, 0, allowSevenAsSunday ? 7 : 6);
};

const hasCompatibleDayFormat = (days: string, weekdays: string, allowBlankDay?: boolean) => {
return !(allowBlankDay && isQuestionMark(days) && isQuestionMark(weekdays));
};

const split = (cron: string): string[] => {
return cron.trim().split(/\s+/);
};

interface Options {
alias: boolean;
seconds: boolean;
allowBlankDay: boolean;
allowSevenAsSunday: boolean;
}

const defaultOptions: Options = {
alias: false,
seconds: false,
allowBlankDay: false,
allowSevenAsSunday: false,
};

export const isValidCron = (cron: string, options?: Partial<Options>): boolean => {
options = { ...defaultOptions, ...options };

const splits = split(cron);

if (splits.length > (options.seconds ? 6 : 5) || splits.length < 5) {
return false;
}

const checks: boolean[] = [];
if (splits.length === 6) {
const seconds = splits.shift();
if (seconds) {
checks.push(hasValidSeconds(seconds));
}
}

// We could only check the steps gradually and return false on the first invalid block,
// However, this won't have any performance impact so why bother for now.
const [minutes, hours, days, months, weekdays] = splits;
checks.push(hasValidMinutes(minutes));
checks.push(hasValidHours(hours));
checks.push(hasValidDays(days, options.allowBlankDay));
checks.push(hasValidMonths(months, options.alias));
checks.push(hasValidWeekdays(weekdays, options.alias, options.allowBlankDay, options.allowSevenAsSunday));
checks.push(hasCompatibleDayFormat(days, weekdays, options.allowBlankDay));

return checks.every(Boolean);
};
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ const mockConnection: WebBackendConnectionRead = {
sourceId: "test-source",
destinationId: "test-destination",
status: ConnectionStatus.active,
schedule: undefined,
scheduleType: "manual",
scheduleData: undefined,
syncCatalog: {
streams: [],
},
Expand Down
Loading

0 comments on commit cfeeeed

Please sign in to comment.