Merge pull request #48 from Kanaries/dev

release: new dataSource module

README.md

@@ -134,15 +134,9 @@ Rath is insipired by several excellent works below:
 
 + Wongsuphasawat, Kanit, et al. "Voyager 2: Augmenting visual analysis with partial view specifications." Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems. ACM, 2017.
 + B. Tang et al, "Extracting top-K insights from multi-dimensional data," in 2017, . DOI: 10.1145/3035918.3035922.
-+ A. Satyanarayan, K. Wongsuphasawat and J. Heer, "Declarative interaction design for data visualization," in 2014, . DOI: 10.1145/2642918.2647360.
++ Vega-Lite: A Grammar of Interactive Graphics. Arvind Satyanarayan, Dominik Moritz, Kanit Wongsuphasawat, Jeffrey Heer. IEEE Trans. Visualization & Comp. Graphics (Proc. InfoVis), 2017
 + Cleveland, W., & McGill, R. (1984). Graphical Perception: Theory, Experimentation, and Application to the Development of Graphical Methods. Journal of the American Statistical Association, 79(387), 531-554. doi:10.2307/2288400
 
 ## Story behind Rath
 
-Rath is original from Mome Raths in *Alice's Adventures in Wonderland*.
-
-<div style="text-align:center;">
-    <img src="https://ch-resources.oss-cn-shanghai.aliyuncs.com/images/mome-raths.png" alt="logo" width="280px" />
-</div>
-
-The word Rath is used in SAO Alicization as name of org. developing Soul Translator (STL) and Under World, which creates A.L.I.C.E.
+The word Rath is used in SAO Alicization as name of org. developing Soul Translator (STL) and Under World, which creates A.L.I.C.E. ('Rath' is original from Mome Raths in *Alice's Adventures in Wonderland*.) I use the Rath as the name of the project for hoping it can create something far more than I could imagine like Alice.

packages/frontend/package.json

@@ -3,6 +3,10 @@
   "version": "0.1.0",
   "private": true,
   "homepage": "./",
+  "prettier": {
+    "tabWidth": 4,
+    "printWidth": 140
+  },
   "dependencies": {
     "@kanaries/web-data-loader": "0.1.0",
     "@tableau/tableau-ui": "^3.0.0",
@@ -13,10 +17,9 @@
     "@types/jest": "^26.0.20",
     "@types/node": "^12.19.12",
     "@types/papaparse": "^5.0.1",
-    "@types/react": "^16.14.2",
-    "@types/react-dom": "^16.9.10",
     "@uifabric/icons": "^7.5.17",
     "@uifabric/react-hooks": "^7.13.9",
+    "ali-react-table": "^2.3.1",
     "chroma-js": "^2.0.6",
     "cube-core": "^2.13.0",
     "datalib": "^1.9.2",
@@ -34,18 +37,20 @@
     "react-json-view": "^1.19.1",
     "react-markdown": "^5.0.3",
     "react-scripts": "4.0.1",
-    "styled-components": "^4.4.1",
+    "styled-components": "^5.2.1",
     "typescript": "^4.1.3",
-    "vega": "^5.6.0",
-    "vega-embed": "^5.1.3",
-    "vega-lite": "^4.0.0",
+    "vega": "^5.19.1",
+    "vega-embed": "^6.15.1",
+    "vega-lite": "^4.17.0",
     "visual-insights": "^0.5.5",
     "web-vitals": "^0.2.4",
     "worker-loader": "^3.0.7"
   },
   "devDependencies": {
+    "@types/react": "^17.0.2",
     "@types/react-beautiful-dnd": "^13.0.0",
-    "@types/styled-components": "4.1.8"
+    "@types/react-dom": "^17.0.1",
+    "@types/styled-components": "^5.1.7"
   },
   "scripts": {
     "start": "react-app-rewired start",

packages/frontend/public/index.html

@@ -7,7 +7,7 @@
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"
-      content="Web site created using create-react-app"
+      content="Kanaries Analytics(Automated data analysis and knowledge discovery)"
     />
     <link rel="apple-touch-icon" href="https://ch-rath.oss-ap-northeast-1.aliyuncs.com/assets/rath.png" />
     <!--

packages/frontend/public/locales/en-US.json

@@ -5,18 +5,40 @@
         "explore": "Explore",
         "dashBoard": "DashBoard",
         "explainer": "Explainer",
-        "editor": "Visual Editor"
+        "editor": "Visual Editor",
+        "support": "Support"
     },
     "preference": {
         "title": "preference"
     },
+    "support": {
+        "title": "How to Get Support",
+        "link": "https://www.yuque.com/docs/share/65511730-af74-46eb-86fe-356c4f52402b?# 《Get Support》"
+    },
     "dataSource": {
         "extractInsight": "Insight Analysis",
         "cleanMethod": "Clean Method",
+        "importData": {
+            "buttonName": "Import Data",
+            "type": {
+                "file": "File",
+                "restful": "RESTFUL",
+                "mysql": "MySQL",
+                "demo": "Demo"
+            },
+            "demo": {
+                "available": "Available Datasets"
+            }
+        },
+        "sampling": {
+            "none": "none",
+            "reservoir": "fixed sample size"
+        },
         "methods": {
             "dropNull": "drop null records",
             "useMode": "replace null with mode",
-            "simpleClean": "simple cleaning"
+            "simpleClean": "simple cleaning",
+            "none": "none(use original data)"
         },
         "tip": "Remember to adjust the fields' types and cleaning strategy before extracting insights.",
         "recordCount": "Number of records {count}",

packages/frontend/public/locales/zh-CN.json

@@ -5,18 +5,40 @@
         "explore": "探索",
         "dashBoard": "报表",
         "explainer": "数据解读",
-        "editor": "自助分析"
+        "editor": "自助分析",
+        "support": "帮助"
     },
     "preference": {
         "title": "个人偏好"
     },
+    "support": {
+        "title": "如何获取帮助/支持",
+        "link": "https://www.yuque.com/docs/share/c1e4077d-c433-422b-9dc9-9486bf772f9c"
+    },
     "dataSource": {
         "extractInsight": "开始分析",
         "cleanMethod": "清洗方法",
+        "importData": {
+            "buttonName": "导入数据",
+            "type": {
+                "file": "文件",
+                "restful": "RESTFUL",
+                "mysql": "MySQL",
+                "demo": "示例数据"
+            },
+            "demo": {
+                "available": "可选数据集"
+            }
+        },
+        "sampling": {
+            "none": "不采样(使用原始数据)",
+            "reservoir": "固定样本大小(数据集较大时使用)"
+        },
         "methods": {
             "dropNull": "删除空值记录",
             "useMode": "使用众数代替空",
-            "simpleClean": "简单的替换"
+            "simpleClean": "简单的替换",
+            "none": "无(使用原始数据)"
         },
         "tip": "记得在提取洞察前先调整数据的清洗策略并配置字段的属性",
         "recordCount": "记录数 {count}",

packages/frontend/src/App.css

@@ -63,11 +63,14 @@ h2, h3 {
 }
 
 .vi-callout{
-  min-Width: 300px;
+  min-Width: 600px;
 }
 .vi-callout-header{
   padding: 18px 24px 12px;
 }
+.vi-callout-close-icon{
+  float: right;
+}
 .vi-callout-title{
   margin: 0;
   font-size: 21px;
@@ -86,7 +89,7 @@ h2, h3 {
   color: #323130;
   font-weight: 300;
 }
-.vi-callout-action{
+.vi-callout-actions{
   position: relative;
   margin-top: 20px;
   width: 100%;

packages/frontend/src/App.tsx

@@ -13,6 +13,7 @@ import VisualEditor from './pages/visualEditor';
 import DataSourceBoard from "./pages/dataSource/index";
 import DashBoardPage from './pages/dashBoard/index';
 import DevPage from './pages/dev';
+import SupportPage from './pages/support/index';
 import UserSettings from './components/userSettings';
 
 require('intl/locale-data/jsonp/en.js')
@@ -45,7 +46,8 @@ function App() {
       intl.get('menu.explore'),
       intl.get('menu.dashBoard'),
       intl.get('menu.explainer'),
-      intl.get('menu.editor')
+      intl.get('menu.editor'),
+      intl.get('menu.support')
     ].map((page, index) => {
       return { title: page, itemKey: 'pivot-' + (index + 1) }
     })
@@ -109,7 +111,7 @@ function App() {
         <DataSourceBoard
           onExtractInsights={() => {
             setPageStatus((draft) => {
-              draft.current.pivotKey = 'pivot-2'
+              draft.current.pivotKey = 'pivot-3'
               draft.show.insightBoard = true
             })
           }}
@@ -125,11 +127,12 @@ function App() {
       {pageStatus.current.pivotKey === 'pivot-4' && <DashBoardPage />}
       {pageStatus.current.pivotKey === 'pivot-5' && <DevPage />}
       {pageStatus.current.pivotKey === 'pivot-6' && <VisualEditor dataSource={state.cookedDataSource} dimensions={state.cookedDimensions} measures={state.cookedMeasures} />}
+      {pageStatus.current.pivotKey === 'pivot-7' && <SupportPage />}
     </div>
   )
 }
 
-export default function() {
+export default function WrappedApp() {
   return (
     <GlobalStateProvider>
       <App />

packages/frontend/src/components/userSettings.tsx

@@ -6,8 +6,7 @@ import {
   ActionButton,
   Toggle,
   DirectionalHint,
-  IDropdownOption,
-  Dropdown
+  IDropdownOption
 } from 'office-ui-fabric-react'
 import { DropdownSelect } from '@tableau/tableau-ui';
 import { useGlobalState } from "../state";

packages/frontend/src/global.ts

@@ -21,4 +21,11 @@ export interface BIField {
 
 export type FieldImpurity = [string, number, number, Field];
 
-export type OperatorType = 'sum' | 'mean' | 'count';
+export type OperatorType = 'sum' | 'mean' | 'count';
+
+export enum IDataSourceType {
+  FILE = 'file',
+  RESTFUL = 'restful',
+  MYSQL = 'mysql',
+  DEMO = 'demo'
+}

packages/frontend/src/index.tsx

@@ -16,8 +16,10 @@ ReactDOM.render(
 // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
 reportWebVitals(console.log);
 
-document.write(
-  unescape(
-    "%3Cspan id='cnzz_stat_icon_1279593444'%3E%3C/span%3E%3Cscript src='https://s9.cnzz.com/z_stat.php%3Fid%3D1279593444%26show%3Dpic1' type='text/javascript'%3E%3C/script%3E"
-  )
-)
+if (process.env.NODE_ENV === 'production') {
+  document.write(
+    unescape(
+      "%3Cspan id='cnzz_stat_icon_1279593444'%3E%3C/span%3E%3Cscript src='https://s9.cnzz.com/z_stat.php%3Fid%3D1279593444%26show%3Dpic1' type='text/javascript'%3E%3C/script%3E"
+    )
+  );
+}

packages/frontend/src/interfaces.ts

@@ -0,0 +1,5 @@
+// define new interfaces here, global.ts is no longer maintained.
+
+export interface IRow {
+    [key: string]: number | string | boolean | null | undefined;
+}

packages/frontend/src/locales.ts

@@ -4,7 +4,7 @@ export const SUPPORT_LANG = [
         value: 'en-US'
     },
     {
-        name: '中文',
+        name: '中文(Chinese)',
         value: 'zh-CN'
     }
 ]

packages/frontend/src/loggers/dataImport.ts

@@ -0,0 +1,32 @@
+import { BIField, Record } from "../global"
+
+const DATA_SOURCE_LOGGER_URL =
+  'https://1423108296428281.cn-hangzhou.fc.aliyuncs.com/2016-08-15/proxy/Rath/dataSourceLogger/'
+
+interface IDataImportInfo {
+    dataType: string;
+    fields: BIField[];
+    dataSource: Record[];
+    name?: string;
+    info?: any;
+    size: number;
+}
+export async function logDataImport (props: IDataImportInfo) {
+    if (process.env.NODE_ENV === 'production') {
+        try {
+          const res = await fetch(DATA_SOURCE_LOGGER_URL, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(props),
+          })
+          const result = await res.json()
+          console.log(result)
+        } catch (error) {
+          console.error(error)
+        }
+    } else {
+        console.log(`Current Env: ${process.env.NODE_ENV}.`, props);
+    }
+}

packages/frontend/src/pages/dataSource/clean.ts

@@ -6,7 +6,10 @@ const { dropNull, simpleClean, useMode } = Cleaner;
 // todo
 // cleanMethodList has redundency.
 // clean method type, cleanData(switch), cleanMethodList should be maintained in one structure.
-export type CleanMethod = 'dropNull' | 'useMode' | 'simpleClean';
+export type CleanMethod = 'dropNull' | 'useMode' | 'simpleClean' | 'none';
+function unClean(dataSource: DataSource) {
+  return [...dataSource];
+}
 export function cleanData (dataSource: DataSource, dimensions: string[], measures: string[], method: CleanMethod): DataSource {
   // hint: dropNull works really bad when we test titanic dataset.
   // useMode fails when there are more null values than normal values;
@@ -18,13 +21,16 @@ export function cleanData (dataSource: DataSource, dimensions: string[], measure
       // eslint-disable-next-line react-hooks/rules-of-hooks
       return useMode(dataSource, dimensions.concat(measures));  
     case 'simpleClean':
-    default:
       return simpleClean(dataSource, dimensions, measures);
+    case 'none':
+    default:
+      return unClean(dataSource);
   }
 }
 
 export const cleanMethodList: Array<{ key: CleanMethod; text: string }> = [
   { key: 'dropNull', text: 'drop null records' },
   { key: 'useMode', text: 'replace null with mode' },
-  { key: 'simpleClean', text: 'simple cleaning' }
+  { key: 'simpleClean', text: 'simple cleaning' },
+  { key: 'none', text: 'none' }
 ]