From 133ec7f97fa03a819fca5c5365ba3ac472f927ea Mon Sep 17 00:00:00 2001 From: AntoineYANG Date: Fri, 16 Jun 2023 11:38:52 +0800 Subject: [PATCH] feat(datasource): auto upload dataset Co-Authored-By: kyusho --- .../rath-client/src/loggers/dataImport.ts | 122 +++++++++++++++++- .../src/pages/dataConnection/create.tsx | 19 ++- .../rath-client/src/store/dataSourceStore.ts | 4 +- 3 files changed, 139 insertions(+), 6 deletions(-) diff --git a/packages/rath-client/src/loggers/dataImport.ts b/packages/rath-client/src/loggers/dataImport.ts index f723404b..84a73ac2 100644 --- a/packages/rath-client/src/loggers/dataImport.ts +++ b/packages/rath-client/src/loggers/dataImport.ts @@ -1,4 +1,7 @@ -import { IMuteFieldBase, IRow } from "../interfaces" +import type { IGeoRole, ISemanticType } from "@kanaries/loa"; +import { IMuteFieldBase, IRawField, IRow } from "../interfaces" +import { request } from "../utils/request"; +import { getMainServiceAddress } from "../utils/user"; const DATA_SOURCE_LOGGER_URL = 'https://1423108296428281.cn-hangzhou.fc.aliyuncs.com/2016-08-15/proxy/Rath/dataSourceLogger/' @@ -54,4 +57,119 @@ export async function dataBackup (file: File) { // eslint-disable-next-line no-console console.log(file) } -} \ No newline at end of file +} + +async function createFileDataset(workspaceName: string, name: string, file: File) { + const { uploadUrl, datasetId } = await request.post<{ + workspaceName: string; fileName: string; name: string; desc: string; meta: any; + }, { + uploadUrl: string; datasetId: string; + }>(getMainServiceAddress('/api/ce/dataset/v2/file/upload'), { + workspaceName, + fileName: file.name, + name, + desc: `Generated by Rath. [${new Date().toLocaleString()}]`, + meta: { + type: 'JSON', + encoding: 'utf8', + fileSize: file.size, + }, + }); + await fetch(uploadUrl, { + method: 'PUT', + body: file, + }); + await request.post(getMainServiceAddress('/api/ce/dataset/v2/file/upload/callback'), { + datasetId, + }); + return { datasetId }; +} + +type IDatasetFieldMeta = { + name: string; + desc: string; + semanticType: ISemanticType; + geoRole: IGeoRole; + dataType: string; +} & ( + | { key: string; fid?: unknown } + | { fid: string; key?: unknown } +); + +async function waitUntilDatasetDraftReady(datasetId: string) { + let retry = 0; + const checkDatasetDraftReady = async () => { + if (retry++ > 10) { + throw new Error('Upload timeout.'); + } + const { step, status, fieldsMeta } = await request.get<{ + datasetId: string; + }, { + step: 'DRAFT' | 'UPLOADED' | 'PROCESSING' | 'PROCESSED' | 'PUBLISHED'; + status: { key: 'ERROR' | 'NORMAL' | 'CANCELED' }; + fieldsMeta: IDatasetFieldMeta[]; + }>(getMainServiceAddress('/api/ce/dataset/v2'), { + datasetId, + }); + if (status.key !== 'NORMAL') { + throw new Error('Upload aborted.'); + } + if (step === 'PROCESSED') { + return fieldsMeta; + } + return new Promise((resolve, reject) => { + setTimeout(() => { + checkDatasetDraftReady().then(resolve, reject); + }, 4_000); + }); + }; + const fieldsMeta = await new Promise((resolve, reject) => { + setTimeout(() => { + checkDatasetDraftReady().then(resolve, reject); + }, 8_000); + }); + return { fieldsMeta }; +} + +async function confirmDatasetDraft(datasetId: string, fields: IMuteFieldBase[], fieldsMeta: IDatasetFieldMeta[]) { + await request.post(getMainServiceAddress('/api/ce/dataset/v2/save'), { + datasetId, + fieldsMeta: fieldsMeta.map((f, i) => { + const fieldKey = f.key || f.fid; + const next = fields[i]; + return { + fid: fieldKey, + key: fieldKey, + name: next?.name || next?.fid || f.name, + desc: '', + semanticType: next?.semanticType || f.semanticType, + geoRole: f?.geoRole || f.geoRole, + dataType: f.dataType, + }; + }), + }); +} + +interface PutCloudDatasetProps { + name: string; + data: IRow[]; + fields: IRawField[]; + workspaceName: string; +} + +export async function putCloudDataset (props: PutCloudDatasetProps) { + if (process.env.NODE_ENV !== 'production') { + return; + } + const { name, data, fields, workspaceName } = props; + try { + const file = new File([JSON.stringify(data)], `${name}.json`, { type: 'application/json' }); + const { datasetId } = await createFileDataset(workspaceName, name, file); + const { fieldsMeta } = await waitUntilDatasetDraftReady(datasetId); + await confirmDatasetDraft(datasetId, fields, fieldsMeta); + // eslint-disable-next-line no-console + console.info('putCloudDataset success. datasetId: ', datasetId); + } catch (error) { + console.warn('putCloudDataset error: ', error); + } +} diff --git a/packages/rath-client/src/pages/dataConnection/create.tsx b/packages/rath-client/src/pages/dataConnection/create.tsx index 8abf44fd..68403c51 100644 --- a/packages/rath-client/src/pages/dataConnection/create.tsx +++ b/packages/rath-client/src/pages/dataConnection/create.tsx @@ -28,6 +28,7 @@ import { useGlobalStore } from '../../store'; import { PIVOT_KEYS } from '../../constants'; import DatabaseConnector from '../dataSource/selection/database'; import { notify } from '../../components/error'; +import { putCloudDataset } from '../../loggers/dataImport'; import FileData from './file'; import DemoData from './demo'; // import RestfulData from './restful'; @@ -54,7 +55,7 @@ const Content = styled.div<{ open: boolean }>` `; const ConnectionCreation: React.FC = (props) => { - const { dataSourceStore, commonStore, megaAutoStore, semiAutoStore } = useGlobalStore(); + const { dataSourceStore, commonStore, megaAutoStore, semiAutoStore, userStore } = useGlobalStore(); const { loading } = dataSourceStore; // const { show, onClose, onDataLoaded, loading, onStartLoading, onLoadingFailed, onDataLoading, toggleLoadingAnimation } = props; @@ -65,17 +66,29 @@ const ConnectionCreation: React.FC = (props) => { commonStore.setAppKey(PIVOT_KEYS.dataSource); }, [commonStore]); + const { workspace } = userStore; + const onSelectDataLoaded = useCallback( (fields: IMuteFieldBase[], dataSource: IRow[], name?: string, tag?: DataSourceTag | undefined, withHistory?: IDBMeta | undefined) => { megaAutoStore.init(); semiAutoStore.init(); - dataSourceStore.loadDataWithInferMetas(dataSource, fields); + const inferTask = dataSourceStore.loadDataWithInferMetas(dataSource, fields); + if (workspace) { + inferTask.then(({ meta }) => { + putCloudDataset({ + workspaceName: workspace.name, + name: name || 'Untitled Dataset', + data: dataSource, + fields: meta, + }); + }); + } if (name && tag !== undefined) { dataSourceStore.setDatasetId(name); setDataStorage(name, fields, dataSource, tag, withHistory); } }, - [dataSourceStore, megaAutoStore, semiAutoStore] + [dataSourceStore, megaAutoStore, semiAutoStore, workspace] ); const onSelectStartLoading = useCallback(() => { diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index 9516b3b2..2ffc2474 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -578,7 +578,7 @@ export class DataSourceStore { this.fieldMetasRef.current = state.fieldMetas } - public async loadDataWithInferMetas (dataSource: IRow[], fields: IMuteFieldBase[], tag?: DataSourceTag | undefined) { + public async loadDataWithInferMetas (dataSource: IRow[], fields: IMuteFieldBase[], tag?: DataSourceTag | undefined): Promise<{ meta: IRawField[] }> { if (fields.length > 0 && dataSource.length > 0) { const metas = await inferMetaService({ dataSource, fields }) await this.rawDataStorage.setAll(dataSource) @@ -604,7 +604,9 @@ export class DataSourceStore { this.mutFields = metas; this.updateDataMetaInIndexedDB() }) + return { meta: metas }; } + return { meta: [] }; } /**