Merge pull request #14 from foghorn-tech/feat-source-dataset_upload-k…

…s-0615

[Feat] auto upload dataset

packages/rath-client/src/loggers/dataImport.ts

@@ -1,4 +1,7 @@
-import { IMuteFieldBase, IRow } from "../interfaces"
+import type { IGeoRole, ISemanticType } from "@kanaries/loa";
+import { IMuteFieldBase, IRawField, IRow } from "../interfaces"
+import { request } from "../utils/request";
+import { getMainServiceAddress } from "../utils/user";
 
 const DATA_SOURCE_LOGGER_URL =
   'https://1423108296428281.cn-hangzhou.fc.aliyuncs.com/2016-08-15/proxy/Rath/dataSourceLogger/'
@@ -54,4 +57,119 @@ export async function dataBackup (file: File) {
     // eslint-disable-next-line no-console
     console.log(file)
   }
-}
+}
+
+async function createFileDataset(workspaceName: string, name: string, file: File) {
+  const { uploadUrl, datasetId } = await request.post<{
+    workspaceName: string; fileName: string; name: string; desc: string; meta: any;
+  }, {
+    uploadUrl: string; datasetId: string;
+  }>(getMainServiceAddress('/api/ce/dataset/v2/file/upload'), {
+    workspaceName,
+    fileName: file.name,
+    name,
+    desc: `Generated by Rath. [${new Date().toLocaleString()}]`,
+    meta: {
+      type: 'JSON',
+      encoding: 'utf8',
+      fileSize: file.size,
+    },
+  });
+  await fetch(uploadUrl, {
+    method: 'PUT',
+    body: file,
+  });
+  await request.post<any, { datasetId: string }>(getMainServiceAddress('/api/ce/dataset/v2/file/upload/callback'), {
+    datasetId,
+  });
+  return { datasetId };
+}
+
+type IDatasetFieldMeta = {
+  name: string;
+  desc: string;
+  semanticType: ISemanticType;
+  geoRole: IGeoRole;
+  dataType: string;
+} & (
+  | { key: string; fid?: unknown }
+  | { fid: string; key?: unknown }
+);
+
+async function waitUntilDatasetDraftReady(datasetId: string) {
+  let retry = 0;
+  const checkDatasetDraftReady = async () => {
+    if (retry++ > 10) {
+      throw new Error('Upload timeout.');
+    }
+    const { step, status, fieldsMeta } = await request.get<{
+      datasetId: string;
+    }, {
+      step: 'DRAFT' | 'UPLOADED' | 'PROCESSING' | 'PROCESSED' | 'PUBLISHED';
+      status: { key: 'ERROR' | 'NORMAL' | 'CANCELED' };
+      fieldsMeta: IDatasetFieldMeta[];
+    }>(getMainServiceAddress('/api/ce/dataset/v2'), {
+      datasetId,
+    });
+    if (status.key !== 'NORMAL') {
+      throw new Error('Upload aborted.');
+    }
+    if (step === 'PROCESSED') {
+      return fieldsMeta;
+    }
+    return new Promise<IDatasetFieldMeta[]>((resolve, reject) => {
+      setTimeout(() => {
+        checkDatasetDraftReady().then(resolve, reject);
+      }, 4_000);
+    });
+  };
+  const fieldsMeta = await new Promise<IDatasetFieldMeta[]>((resolve, reject) => {
+    setTimeout(() => {
+      checkDatasetDraftReady().then(resolve, reject);
+    }, 8_000);
+  });
+  return { fieldsMeta };
+}
+
+async function confirmDatasetDraft(datasetId: string, fields: IMuteFieldBase[], fieldsMeta: IDatasetFieldMeta[]) {
+  await request.post(getMainServiceAddress('/api/ce/dataset/v2/save'), {
+    datasetId,
+    fieldsMeta: fieldsMeta.map((f, i) => {
+      const fieldKey = f.key || f.fid;
+      const next = fields[i];
+      return {
+        fid: fieldKey,
+        key: fieldKey,
+        name: next?.name || next?.fid || f.name,
+        desc: '',
+        semanticType: next?.semanticType || f.semanticType,
+        geoRole: f?.geoRole || f.geoRole,
+        dataType: f.dataType,
+      };
+    }),
+  });
+}
+
+interface PutCloudDatasetProps {
+  name: string;
+  data: IRow[];
+  fields: IRawField[];
+  workspaceName: string;
+}
+
+export async function putCloudDataset (props: PutCloudDatasetProps) {
+  if (process.env.NODE_ENV !== 'production') {
+    return;
+  }
+  const { name, data, fields, workspaceName } = props;
+  try {
+    const file = new File([JSON.stringify(data)], `${name}.json`, { type: 'application/json' });
+    const { datasetId } = await createFileDataset(workspaceName, name, file);
+    const { fieldsMeta } = await waitUntilDatasetDraftReady(datasetId);
+    await confirmDatasetDraft(datasetId, fields, fieldsMeta);
+    // eslint-disable-next-line no-console
+    console.info('putCloudDataset success. datasetId: ', datasetId);
+  } catch (error) {
+    console.warn('putCloudDataset error: ', error);
+  }
+}

packages/rath-client/src/pages/dataConnection/create.tsx

@@ -28,6 +28,7 @@ import { useGlobalStore } from '../../store';
 import { PIVOT_KEYS } from '../../constants';
 import DatabaseConnector from '../dataSource/selection/database';
 import { notify } from '../../components/error';
+import { putCloudDataset } from '../../loggers/dataImport';
 import FileData from './file';
 import DemoData from './demo';
 // import RestfulData from './restful';
@@ -54,7 +55,7 @@ const Content = styled.div<{ open: boolean }>`
 `;
 
 const ConnectionCreation: React.FC<ConnectionCreationProps> = (props) => {
-    const { dataSourceStore, commonStore, megaAutoStore, semiAutoStore } = useGlobalStore();
+    const { dataSourceStore, commonStore, megaAutoStore, semiAutoStore, userStore } = useGlobalStore();
     const { loading } = dataSourceStore;
     // const { show, onClose, onDataLoaded, loading, onStartLoading, onLoadingFailed, onDataLoading, toggleLoadingAnimation } = props;
 
@@ -65,17 +66,29 @@ const ConnectionCreation: React.FC<ConnectionCreationProps> = (props) => {
         commonStore.setAppKey(PIVOT_KEYS.dataSource);
     }, [commonStore]);
 
+    const { workspace } = userStore;
+
     const onSelectDataLoaded = useCallback(
         (fields: IMuteFieldBase[], dataSource: IRow[], name?: string, tag?: DataSourceTag | undefined, withHistory?: IDBMeta | undefined) => {
             megaAutoStore.init();
             semiAutoStore.init();
-            dataSourceStore.loadDataWithInferMetas(dataSource, fields);
+            const inferTask = dataSourceStore.loadDataWithInferMetas(dataSource, fields);
+            if (workspace) {
+                inferTask.then(({ meta }) => {
+                    putCloudDataset({
+                        workspaceName: workspace.name,
+                        name: name || 'Untitled Dataset',
+                        data: dataSource,
+                        fields: meta,
+                    });
+                });
+            }
             if (name && tag !== undefined) {
                 dataSourceStore.setDatasetId(name);
                 setDataStorage(name, fields, dataSource, tag, withHistory);
             }
         },
-        [dataSourceStore, megaAutoStore, semiAutoStore]
+        [dataSourceStore, megaAutoStore, semiAutoStore, workspace]
     );
 
     const onSelectStartLoading = useCallback(() => {

packages/rath-client/src/store/dataSourceStore.ts

@@ -578,7 +578,7 @@ export class DataSourceStore {
         this.fieldMetasRef.current = state.fieldMetas
     } 
 
-    public async loadDataWithInferMetas (dataSource: IRow[], fields: IMuteFieldBase[], tag?: DataSourceTag | undefined) {
+    public async loadDataWithInferMetas (dataSource: IRow[], fields: IMuteFieldBase[], tag?: DataSourceTag | undefined): Promise<{ meta: IRawField[] }> {
         if (fields.length > 0 && dataSource.length > 0) {
             const metas = await inferMetaService({ dataSource, fields })
             await this.rawDataStorage.setAll(dataSource)
@@ -604,7 +604,9 @@ export class DataSourceStore {
                 this.mutFields = metas;
                 this.updateDataMetaInIndexedDB()
             })
+            return { meta: metas };
         }
+        return { meta: [] };
     }
 
     /**