Skip to content

Commit

Permalink
Add exitOnNoData flag to fetchData (#4399)
Browse files Browse the repository at this point in the history
  • Loading branch information
IgorNadj authored Mar 27, 2023
1 parent a56b58f commit 0018752
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ import { Row } from '../../../types';
import { createJoin } from './createJoin';
import { TransformTable } from '../../table';
import { TransformParser } from '../../parser';
import { ExitWithNoDataSignal } from '../../transform';

type FetchParams = {
dataTableCode: string;
parameters: Record<string, unknown>;
join: (existingRows: Row[], newRows: Row[]) => Row[];
exitOnNoData?: boolean;
};

export const paramsValidator = yup.object().shape({
Expand All @@ -25,10 +27,11 @@ export const paramsValidator = yup.object().shape({
newDataColumn: yup.string().required(),
}),
),
exitOnNoData: yup.boolean().default(true),
});

const fetchData = async (table: TransformTable, params: FetchParams, context: Context) => {
const { dataTableCode, parameters, join } = params;
const { dataTableCode, parameters, join, exitOnNoData } = params;
const parser = new TransformParser(table, context);

const parsedParameters = Object.fromEntries(
Expand All @@ -47,6 +50,11 @@ const fetchData = async (table: TransformTable, params: FetchParams, context: Co
);

const newRows = response.data as Row[];

if (newRows.length === 0 && exitOnNoData) {
throw new ExitWithNoDataSignal();
}

const existingColumns = table.getColumns();
const newColumns = Array.from(new Set(newRows.map(Object.keys).flat())).filter(
column => !existingColumns.includes(column),
Expand All @@ -59,13 +67,14 @@ const fetchData = async (table: TransformTable, params: FetchParams, context: Co
};

const buildParams = (params: unknown): FetchParams => {
const { dataTableCode, parameters, join } = paramsValidator.validateSync(params);
const { dataTableCode, parameters, join, exitOnNoData } = paramsValidator.validateSync(params);

const joinFunction = createJoin(join);
return {
dataTableCode,
parameters,
join: joinFunction,
exitOnNoData,
};
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ const transform = async (table: TransformTable, transformSteps: BuiltTransformPa
try {
transformedTable = await transformStep.apply(transformedTable);
} catch (e) {
if (e instanceof ExitWithNoDataSignal) {
// Return a no data result
return new TransformTable();
}
const titlePart = transformStep.title ? ` (${transformStep.title})` : '';
const errorMessagePrefix = `Error in transform[${i + 1}]${titlePart}: `;
e.message = `${errorMessagePrefix}${(e as Error).message}`;
Expand Down Expand Up @@ -82,3 +86,8 @@ export const buildTransform = (params: unknown, context: Context) => {
const builtParams = validatedParams.map(param => buildParams(param, context));
return (table: TransformTable) => transform(table, builtParams);
};

/**
* A signal to exit the transform steps early
*/
export class ExitWithNoDataSignal extends Error {}

0 comments on commit 0018752

Please sign in to comment.