From 5c432570817ff80d859c169321223a51c8cf70a8 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Thu, 14 Jul 2022 23:50:39 +0800 Subject: [PATCH 01/15] feat: refactor the arc and design of semi-auto --- .../rath-client/public/locales/en-US.json | 7 +- .../rath-client/public/locales/zh-CN.json | 7 +- packages/rath-client/src/App.css | 2 +- .../pages/semiAutomation/focusZone/index.tsx | 97 ++++++ .../semiAutomation/focusZone/mainCanvas.tsx | 76 +++++ .../focusZone/miniFloatCanvas.tsx | 80 +++++ .../src/pages/semiAutomation/index.tsx | 294 ++++------------ .../predictZone/featSegment.tsx | 61 ++++ .../predictZone/filterSegment.tsx | 68 ++++ .../semiAutomation/predictZone/index.tsx | 28 ++ .../predictZone/pattSegment.tsx | 87 +++++ .../pages/semiAutomation/predictZone/tmp.txt | 313 ++++++++++++++++++ .../src/pages/semiAutomation/settings.tsx | 15 +- .../src/store/discovery/localTypes.ts | 32 ++ .../src/store/discovery/mainStore.ts | 269 ++++++++++++++- packages/rath-client/src/store/index.tsx | 2 +- packages/rath-client/src/utils/index.ts | 12 + 17 files changed, 1202 insertions(+), 248 deletions(-) create mode 100644 packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx create mode 100644 packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt create mode 100644 packages/rath-client/src/store/discovery/localTypes.ts diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index 70118e22..a8609c4c 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -297,7 +297,12 @@ "lite": "Lite mode(fast)", "strict": "Strict mode" }, - "loadMore": "load more" + "loadMore": "load more", + "associate": { + "patterns": "Associated Patterns", + "features": "Associated Features", + "filters": "Associated Subsets" + } } } } \ No newline at end of file diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 633270ff..1399cc58 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -297,7 +297,12 @@ "lite": "轻量模式(近似算法、更快)", "strict": "严格模式(效果更好)" }, - "loadMore": "加载更多" + "loadMore": "加载更多", + "associate": { + "patterns": "关联规律", + "features": "关联维度", + "filters": "关联子集" + } } } } \ No newline at end of file diff --git a/packages/rath-client/src/App.css b/packages/rath-client/src/App.css index 60fd7ec1..ece34a40 100644 --- a/packages/rath-client/src/App.css +++ b/packages/rath-client/src/App.css @@ -59,7 +59,7 @@ h2, h3 { padding: 28px; border-radius: 2px; box-shadow: 0 1.6px 3.6px 0 rgba(0,0,0,0.132), 0 0.3px 0.9px 0 rgba(0,0,0,0.108); - margin-bottom: 28px; + margin-bottom: 1em; animation-duration: 0.5s; animation-name: showCard; } diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx new file mode 100644 index 00000000..8cfc0544 --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx @@ -0,0 +1,97 @@ +import { observer } from 'mobx-react-lite'; +import { DefaultButton, PrimaryButton } from 'office-ui-fabric-react'; +import React, { useCallback } from 'react'; +import { IFieldMeta } from '../../../interfaces'; +import { useGlobalStore } from '../../../store'; +import ViewField from '../../megaAutomation/vizOperation/viewField'; +import { MainViewContainer } from '../components'; +import MainCanvas from './mainCanvas'; +import intl from 'react-intl-universal'; +import MiniFloatCanvas from './miniFloatCanvas'; + +const BUTTON_STYLE = { marginRight: '1em' } + +const FocusZone: React.FC = props => { + const { discoveryMainStore } = useGlobalStore(); + const { mainView, compareView, showMiniFloatView, autoAsso } = discoveryMainStore; + const advicePureFeature = useCallback(() => { + discoveryMainStore.featAssociate() + }, [discoveryMainStore]) + const assViews = useCallback(() => { + discoveryMainStore.pattAssociate(); + discoveryMainStore.featAssociate(); + }, [discoveryMainStore]) + const recommandFilter = useCallback(() => { + discoveryMainStore.filterAssociate(); + }, [discoveryMainStore]) + const explainDiff = useCallback(() => { + if (mainView && compareView) { + discoveryMainStore.explainViewDiff(mainView, compareView); + } + }, [mainView, compareView, discoveryMainStore]) + return + {mainView && showMiniFloatView && } +
+
+ {mainView && } +
+
+ {compareView && } +
+
+
+ { + mainView && mainView.fields.map((f: IFieldMeta) => { + discoveryMainStore.removeMainViewField(f.fid) + }} + />) + } +
+
+ { + mainView && mainView.filters && mainView.filters.map(f => { + discoveryMainStore.removeMainViewFilter(f.field.fid) + }} + />) + } +
+
+ { + !autoAsso.pattViews && + } + { + !autoAsso.featViews && + } + + { + !autoAsso.filterViews && + } +
+
+} + +export default observer(FocusZone); diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx new file mode 100644 index 00000000..bc596c0f --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx @@ -0,0 +1,76 @@ +import { observer } from 'mobx-react-lite'; +import { Resizable } from 're-resizable'; +import React, { useMemo } from 'react'; +import ReactVega from '../../../components/react-vega'; +import { IPattern } from '../../../dev'; +import { IResizeMode, IRow } from '../../../interfaces'; +import { distVis } from '../../../queries/distVis'; +import { labDistVis } from '../../../queries/labdistVis'; +import { useGlobalStore } from '../../../store'; +import { applyFilter } from '../utils'; + +interface MainCanvasProps{ + pined: IPattern; +} +const MainCanvas: React.FC = props => { + const { pined } = props; + const { discoveryMainStore } = useGlobalStore() + const { settings, mainVizSetting, dataSource } = discoveryMainStore; + const { vizAlgo } = settings; + + const { resize, debug, interactive } = mainVizSetting + const { width, height, mode } = resize; + const mainViewData = useMemo(() => { + if (pined) return applyFilter(dataSource, pined.filters) + return [] + }, [dataSource, pined]) + + const spec = useMemo(() => { + if (vizAlgo === 'lite') { + return distVis({ + resizeMode: mode, + width, + height, + pattern: pined, + interactive + }) + } else { + return labDistVis({ + resizeMode: mode, + pattern: pined, + width, + height, + dataSource + }) + } + }, [mode, height, width, interactive, pined, vizAlgo, dataSource]) + + const enableResize = (mode === IResizeMode.control && spec.encoding && !spec.encoding.column && !spec.encoding.row) + + if (enableResize) { + return { + discoveryMainStore.updateMainVizSettings(s => { + s.resize.width = s.resize.width + d.width; + s.resize.height = s.resize.height + d.height + }) + }}> + + + } + return +} + +export default observer(MainCanvas); diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx new file mode 100644 index 00000000..d84bb288 --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx @@ -0,0 +1,80 @@ +import { observer } from 'mobx-react-lite'; +import { CommandBarButton, IconButton } from 'office-ui-fabric-react'; +import React, { useMemo, useState } from 'react'; +import styled from 'styled-components'; + +import ReactVega from '../../../components/react-vega'; +import { IPattern } from '../../../dev'; +import { IRow } from '../../../interfaces'; +import { distVis } from '../../../queries/distVis'; +import { labDistVis } from '../../../queries/labdistVis'; +import { useGlobalStore } from '../../../store'; +import { applyFilter } from '../utils'; + +const FloatContainer = styled.div<{hide: boolean}>` + position: fixed; + right: 1px; + top: 30%; + z-index: 99; + padding: ${props => props.hide ? '5px' : '1em'}; + background-color: #fff; + box-shadow: 0 10px 8px rgba(0, 0, 0, 0.05), 0 4px 3px rgba(0, 0, 0, 0.01); + .actions { + margin-bottom: 6px; + } +` + +interface MiniFloatCanvasProps{ + pined: IPattern; +} +const MiniFloatCanvas: React.FC = props => { + const { pined } = props; + const { discoveryMainStore } = useGlobalStore() + const { settings, mainVizSetting, dataSource } = discoveryMainStore; + const { vizAlgo } = settings; + const [hide, setHide] = useState(false); + + const { debug } = mainVizSetting + const mainViewData = useMemo(() => { + if (pined) return applyFilter(dataSource, pined.filters) + return [] + }, [dataSource, pined]) + + const spec = useMemo(() => { + return vizAlgo === 'lite' ? distVis({ pattern: pined }) : labDistVis({ + pattern: pined, + dataSource + }) + }, [pined, vizAlgo, dataSource]) + + return +
+ { + !hide && { + setHide(v => !v) + }} + /> + } + { + hide && { + setHide(v => !v) + }} + /> + } +
+ { + !hide && + } +
+} + +export default observer(MiniFloatCanvas); diff --git a/packages/rath-client/src/pages/semiAutomation/index.tsx b/packages/rath-client/src/pages/semiAutomation/index.tsx index 55729ac2..9b5da3f3 100644 --- a/packages/rath-client/src/pages/semiAutomation/index.tsx +++ b/packages/rath-client/src/pages/semiAutomation/index.tsx @@ -1,5 +1,5 @@ import { observer } from 'mobx-react-lite'; -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { IFilter, IPattern } from '../../dev'; import { useGlobalStore } from '../../store'; import { distVis } from '../../queries/distVis' @@ -14,61 +14,63 @@ import { labDistVis } from '../../queries/labdistVis'; import { notify } from '../../components/error'; import intl from 'react-intl-universal'; import MainCanvas from './mainCanvas'; +import FocusZone from './focusZone'; +import PredictZone from './predictZone'; +import { throttle } from '../../utils'; const BUTTON_STYLE = { marginRight: '1em' } function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { if (!filters || filters.length === 0) return dataSource; - return dataSource.filter(row => { - return filters.every(f => f.values.includes(row[f.field.fid])) - }) + const subset: IRow[] = []; + const filterValueSetList: Array> = []; + for (let i = 0; i < filters.length; i++) { + filterValueSetList.push(new Set(filters[i].values)) + } + for (let i = 0; i < dataSource.length; i++) { + for (let j = 0; j < filters.length; j++) { + if (filterValueSetList[j].has(dataSource[i][filters[j].field.fid])) { + subset.push(dataSource[i]) + } + } + } + return subset + // return dataSource.filter(row => { + // return filters.every(f => f.values.includes(row[f.field.fid])) + // }) } const RENDER_BATCH_SIZE = 5; -const PatternPage: React.FC = props => { +const PatternPage: React.FC = () => { + const focusZoneContainer = useRef(null); const { dataSourceStore, discoveryMainStore } = useGlobalStore(); - const { fieldMetas, cleanedData } = dataSourceStore; - const [views, setViews] = useState([]) + // const { fieldMetas, cleanedData } = dataSourceStore; + const [views, setViews] = useState([]); const [pined, setPined] = useState(null); const [computing, setComputing] = useState(false); const [renderAmount, setRenderAmount] = useState(RENDER_BATCH_SIZE); const [mergeView, setMergeView] = useState(null); - const { mainVizSetting } = discoveryMainStore; + const { mainVizSetting, + featViews, + filterViews, + pattViews, + fieldMetas, + dataSource, + featSpecList, + pattSpecList, + filterSpecList + } = discoveryMainStore; + const vizRecSys = discoveryMainStore.settings.vizAlgo; useEffect(() => { - // core.init(cleanedData, fieldMetas) - footmanEngineService({ - dataSource: cleanedData, - fields: fieldMetas, - task: 'univar', - }, 'local').then(res => { - setViews(res); - }).catch(console.error) - // const patterns = core.searchPatterns(); - // setViews(patterns); - }, [fieldMetas, cleanedData]) - - const specs = useMemo(() => { - return views.map(view => distVis({ pattern: view })) - }, [views]) + discoveryMainStore.initAssociate(); + }, [fieldMetas, dataSource]) const assViews = useCallback((view: IPattern) => { - setComputing(true) - footmanEngineService({ - dataSource: cleanedData, - fields: fieldMetas, - task: 'patterns', - props: view - }).then(res => { - setViews(res); - setComputing(false) - setRenderAmount(RENDER_BATCH_SIZE) - }).catch(err => { - console.error(err) - setComputing(false) - }); - }, [fieldMetas, cleanedData]) + discoveryMainStore.pattAssociate(); + discoveryMainStore.featAssociate(); + }, []) const adviceCompareFeature = useCallback(() => { if (pined === null || mergeView === null) return; @@ -79,7 +81,7 @@ const PatternPage: React.FC = props => { // props: // }) footmanEngineService({ - dataSource: cleanedData, + dataSource, fields: fieldMetas, task: 'comparison', props: [pined, mergeView] @@ -103,20 +105,11 @@ const PatternPage: React.FC = props => { content: `${err}` }) }) - }, [pined, mergeView, cleanedData, fieldMetas]) + }, [pined, mergeView, dataSource, fieldMetas]) const advicePureFeature = useCallback(() => { - if (pined === null) return; - footmanEngineService({ - dataSource: cleanedData, - fields: fieldMetas, - task: 'featureSelection', - props: pined - }).then(ans => { - setViews(ans) - setRenderAmount(RENDER_BATCH_SIZE) - }).catch(console.error); - }, [pined, cleanedData, fieldMetas]) + discoveryMainStore.featAssociate() + }, []) const removeFromPined = useCallback((fid: string) => { if (pined === null) return; @@ -132,18 +125,8 @@ const PatternPage: React.FC = props => { }, [pined]) const recommandFilter = useCallback(() => { - if (pined === null) return; - footmanEngineService({ - task: 'filterSelection', - fields: fieldMetas, - dataSource: cleanedData, - props: pined - }).then(res => { - setViews(res); - setRenderAmount(RENDER_BATCH_SIZE) - }).catch(console.error); - - }, [cleanedData, fieldMetas, pined]) + discoveryMainStore.filterAssociate(); + }, []) const removePinedFilter = useCallback((filterField: IFieldMeta) => { setPined(p => { @@ -162,12 +145,28 @@ const PatternPage: React.FC = props => { } }, [pined, assViews]) - const vizRecSys = discoveryMainStore.settings.vizAlgo; + useEffect(() => { + const ele = document.querySelector('.main-app-content'); + const callback = throttle((e: Event) => { + if (focusZoneContainer.current && ele) { + if (focusZoneContainer.current.offsetTop + focusZoneContainer.current.offsetHeight < ele.scrollTop) { + discoveryMainStore.setShowMiniFloatView(true) + } else { + discoveryMainStore.setShowMiniFloatView(false); + } + } + }, 300) + ele!.addEventListener('scroll', callback) + return () => { + if (ele) { + ele.removeEventListener('scroll', callback); + } + } + }, [discoveryMainStore]) return
-
- { computing && 'yes' } +
{ discoveryMainStore.setShowSettings(true); }} /> - -

{intl.get('discovery.main.mainView')}

-
-
- { - pined !== null && - } -
- - {mergeView !== null &&
- { - vizRecSys === 'lite' && - } - { - vizRecSys === 'strict' && - } -
} - -
-
- { - pined && pined.fields.map((f: IFieldMeta) => { removeFromPined(f.fid) }} - />) - } -
-
- { - pined && pined.filters && pined.filters.map(f => { - removePinedFilter(f.field) - }} - />) - } -
-
- { - if (pined) { - assViews(pined) - } - }} - /> - - - -
-
-
- - { - specs.slice(0, renderAmount).map((spec, i) =>
- { - computing && - - - } - - { - setPined(views[i]) - }} - /> - { - setMergeView(views[i]) - discoveryMainStore.updateMainVizSettings(s => { - s.resize.mode = IResizeMode.auto - }) - }} - /> - {/* { - setPined(views[i]) - }} - /> - { - setMergeView(views[i]) - discoveryMainStore.updateMainVizSettings(s => { - s.resize.mode = IResizeMode.auto - }) - }} - /> */} - -
- { - vizRecSys === 'lite' && - } - { - vizRecSys === 'strict' && - } -
- { - views[i].filters &&
-

filters

- {views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} -
- } -
) - } -
- = specs.length} + + {/* = specs.length} style={{ marginTop: '5px' }} text={intl.get('discovery.main.loadMore')} onClick={() => { setRenderAmount(a => a + RENDER_BATCH_SIZE) }} - /> + /> */}
+
} diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx new file mode 100644 index 00000000..30072a1e --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx @@ -0,0 +1,61 @@ +import React, { useCallback } from 'react'; +import { observer } from 'mobx-react-lite'; +import intl from 'react-intl-universal'; +import { CommandButton, DefaultButton, Spinner, Stack } from 'office-ui-fabric-react'; + +import { useGlobalStore } from '../../../store'; +import { AssoContainer, LoadingLayer } from '../components'; +import ReactVega from '../../../components/react-vega'; +import { applyFilter } from '../utils'; + +const FeatSegment: React.FC = () => { + const { discoveryMainStore } = useGlobalStore(); + const { featSpecList, computing, featViews, mainVizSetting, dataSource } = discoveryMainStore; + const loadMore = useCallback(() => { + discoveryMainStore.increaseRenderAmount('featViews'); + }, [discoveryMainStore]) + if (featViews.views.length === 0) return
+ return
+

{intl.get('discovery.main.associate.features')}

+ + { + featSpecList.map((spec, i) =>
+ { + computing && + + + } + + { + discoveryMainStore.updateMainView(featViews.views[i]) + }} + /> + +
+ +
+ { + featViews.views[i].filters &&
+

filters

+ {featViews.views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} +
+ } +
) + } +
+ = featViews.views.length} + style={{ marginTop: '8px' }} + text={intl.get('discovery.main.loadMore')} + onClick={loadMore} + /> +
+} + +export default observer(FeatSegment); diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx new file mode 100644 index 00000000..315d8dcb --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx @@ -0,0 +1,68 @@ +import React, { useCallback } from 'react'; +import { observer } from 'mobx-react-lite'; +import intl from 'react-intl-universal'; +import { CommandButton, DefaultButton, Spinner, Stack } from 'office-ui-fabric-react'; + +import { useGlobalStore } from '../../../store'; +import { AssoContainer, LoadingLayer } from '../components'; +import ReactVega from '../../../components/react-vega'; +import { applyFilter } from '../utils'; + +const FilterSegment: React.FC = () => { + const { discoveryMainStore } = useGlobalStore(); + const { filterSpecList, computing, filterViews, mainVizSetting, dataSource } = discoveryMainStore; + const loadMore = useCallback(() => { + discoveryMainStore.increaseRenderAmount('featViews'); + }, [discoveryMainStore]) + if (filterViews.views.length === 0) return
+ return
+

{intl.get('discovery.main.associate.filters')}

+ + { + filterSpecList.map((spec, i) =>
+ { + computing && + + + } + + { + discoveryMainStore.updateMainView(filterViews.views[i]) + }} + /> + { + discoveryMainStore.updateCompareView(filterViews.views[i]) + }} + /> + +
+ +
+ { + filterViews.views[i].filters &&
+

filters

+ {filterViews.views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} +
+ } +
) + } +
+ = filterViews.views.length} + style={{ marginTop: '8px' }} + text={intl.get('discovery.main.loadMore')} + onClick={loadMore} + /> +
+} + +export default observer(FilterSegment); diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx new file mode 100644 index 00000000..3309af71 --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx @@ -0,0 +1,28 @@ +import React, { useEffect } from 'react'; +import { observer } from 'mobx-react-lite'; + +import { useGlobalStore } from '../../../store'; +import PattSegment from './pattSegment'; +import FeatSegment from './featSegment'; +import FilterSegment from './filterSegment'; + + +const PredictZone: React.FC = props => { + const { discoveryMainStore } = useGlobalStore(); + const { dataSource, fieldMetas, mainView, autoAsso } = discoveryMainStore; + const { pattViews: autoPatt, featViews: autoFeat, filterViews: autoFilter } = autoAsso; + useEffect(() => { + if (mainView) { + autoPatt && discoveryMainStore.pattAssociate(); + autoFeat && discoveryMainStore.featAssociate(); + autoFilter && discoveryMainStore.filterAssociate(); + } + }, [dataSource, fieldMetas, mainView, autoFeat, autoPatt, autoFilter, discoveryMainStore]) + return
+ + + +
+} + +export default observer(PredictZone); diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx new file mode 100644 index 00000000..a1b4a0e4 --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx @@ -0,0 +1,87 @@ +import React, { useCallback } from 'react'; +import { observer } from 'mobx-react-lite'; +import intl from 'react-intl-universal'; +import { CommandButton, DefaultButton, Spinner, Stack } from 'office-ui-fabric-react'; + +import { useGlobalStore } from '../../../store'; +import { AssoContainer, LoadingLayer } from '../components'; +import ReactVega from '../../../components/react-vega'; +import { applyFilter } from '../utils'; + +const PattSegment: React.FC = () => { + const { discoveryMainStore } = useGlobalStore(); + const { pattSpecList, computing, pattViews, mainVizSetting, dataSource } = discoveryMainStore; + const loadMore = useCallback(() => { + discoveryMainStore.increaseRenderAmount('featViews'); + }, [discoveryMainStore]) + if (pattViews.views.length === 0) return
+ return
+

{intl.get('discovery.main.associate.patterns')}

+ + { + pattSpecList.map((spec, i) =>
+ { + computing && + + + } + + { + discoveryMainStore.updateMainView(pattViews.views[i]) + }} + /> + {/* { + discoveryMainStore.updateMainView(pattViews.views[i]) + setMergeView(views[i]) + discoveryMainStore.updateMainVizSettings(s => { + s.resize.mode = IResizeMode.auto + }) + }} + /> */} + {/* { + setPined(views[i]) + }} + /> + { + setMergeView(views[i]) + discoveryMainStore.updateMainVizSettings(s => { + s.resize.mode = IResizeMode.auto + }) + }} + /> */} + +
+ +
+ { + pattViews.views[i].filters &&
+

filters

+ {pattViews.views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} +
+ } +
) + } +
+ = pattViews.views.length} + style={{ marginTop: '8px' }} + text={intl.get('discovery.main.loadMore')} + onClick={loadMore} + /> +
+} + +export default observer(PattSegment); diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt b/packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt new file mode 100644 index 00000000..2391aa41 --- /dev/null +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt @@ -0,0 +1,313 @@ +import { observer } from 'mobx-react-lite'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { IFilter, IPattern } from '../../dev'; +import { useGlobalStore } from '../../store'; +import { distVis } from '../../queries/distVis' +import ReactVega from '../../components/react-vega'; +import { DefaultButton, Stack, PrimaryButton, ActionButton, CommandButton, Spinner } from 'office-ui-fabric-react'; +import { AssoContainer, LoadingLayer, MainViewContainer } from './components'; +import ViewField from '../megaAutomation/vizOperation/viewField'; +import { IFieldMeta, IResizeMode, IRow } from '../../interfaces'; +import { footmanEngineService } from '../../service'; +import Settings from './settings'; +import { labDistVis } from '../../queries/labdistVis'; +import { notify } from '../../components/error'; +import intl from 'react-intl-universal'; +import MainCanvas from './mainCanvas'; + +const BUTTON_STYLE = { marginRight: '1em' } + +function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { + if (!filters || filters.length === 0) return dataSource; + const subset: IRow[] = []; + const filterValueSetList: Array> = []; + for (let i = 0; i < filters.length; i++) { + filterValueSetList.push(new Set(filters[i].values)) + } + for (let i = 0; i < dataSource.length; i++) { + for (let j = 0; j < filters.length; j++) { + if (filterValueSetList[j].has(dataSource[i][filters[j].field.fid])) { + subset.push(dataSource[i]) + } + } + } + return subset + // return dataSource.filter(row => { + // return filters.every(f => f.values.includes(row[f.field.fid])) + // }) +} + +const RENDER_BATCH_SIZE = 5; + +const PatternPage: React.FC = () => { + const { dataSourceStore, discoveryMainStore } = useGlobalStore(); + // const { fieldMetas, cleanedData } = dataSourceStore; + const [views, setViews] = useState([]); + const [pined, setPined] = useState(null); + const [computing, setComputing] = useState(false); + const [renderAmount, setRenderAmount] = useState(RENDER_BATCH_SIZE); + const [mergeView, setMergeView] = useState(null); + const { mainVizSetting, + featViews, + filterViews, + pattViews, + fieldMetas, + dataSource, + featSpecList, + pattSpecList, + filterSpecList + } = discoveryMainStore; + const vizRecSys = discoveryMainStore.settings.vizAlgo; + + useEffect(() => { + discoveryMainStore.initAssociate(); + }, [fieldMetas, dataSource]) + + const assViews = useCallback((view: IPattern) => { + discoveryMainStore.pattAssociate(); + discoveryMainStore.featAssociate(); + }, []) + + const adviceCompareFeature = useCallback(() => { + if (pined === null || mergeView === null) return; + // footmanEngineService({ + // dataSource: cleanedData, + // fields: fieldMetas, + // task: 'featureSelection', + // props: + // }) + footmanEngineService({ + dataSource, + fields: fieldMetas, + task: 'comparison', + props: [pined, mergeView] + }).then(res => { + if (res !== null) { + setViews([ + { + ...pined, + fields: [...pined.fields, ...res.features] + }, + { + ...mergeView, + fields: [...mergeView.fields, ...res.features] + }, + ]) + } + }).catch(err => { + notify({ + title: 'comparsion error', + type: 'error', + content: `${err}` + }) + }) + }, [pined, mergeView, dataSource, fieldMetas]) + + const advicePureFeature = useCallback(() => { + discoveryMainStore.featAssociate() + }, []) + + const removeFromPined = useCallback((fid: string) => { + if (pined === null) return; + const fields: IFieldMeta[] = [...pined.fields]; + const targetIndex = fields.findIndex(f => f.fid === fid); + if (targetIndex > -1) { + fields.splice(targetIndex, 1) + setPined({ + ...pined, + fields: fields + }) + } + }, [pined]) + + const recommandFilter = useCallback(() => { + discoveryMainStore.filterAssociate(); + }, []) + + const removePinedFilter = useCallback((filterField: IFieldMeta) => { + setPined(p => { + if (!p?.filters) return p; + return { + ...p, + filters: p.filters.filter(f => f.field.fid !== filterField.fid) + } + }) + }, []) + + useEffect(() => { + if (pined) { + assViews(pined) + setRenderAmount(RENDER_BATCH_SIZE) + } + }, [pined, assViews]) + + return
+ +
+ { + discoveryMainStore.setShowSettings(true); + }} + /> + +

{intl.get('discovery.main.mainView')}

+
+
+ { + pined !== null && + } +
+ + {mergeView !== null &&
+ {/* { + vizRecSys === 'lite' && + } */} + {/* { + vizRecSys === 'strict' && + } */} +
} + +
+
+ { + ma && ma.fields.map((f: IFieldMeta) => { removeFromPined(f.fid) }} + />) + } +
+
+ { + pined && pined.filters && pined.filters.map(f => { + removePinedFilter(f.field) + }} + />) + } +
+
+ { + if (pined) { + assViews(pined) + } + }} + /> + + + +
+
+
+ + { + specs.slice(0, renderAmount).map((spec, i) =>
+ { + computing && + + + } + + { + setPined(views[i]) + }} + /> + { + setMergeView(views[i]) + discoveryMainStore.updateMainVizSettings(s => { + s.resize.mode = IResizeMode.auto + }) + }} + /> + {/* { + setPined(views[i]) + }} + /> + { + setMergeView(views[i]) + discoveryMainStore.updateMainVizSettings(s => { + s.resize.mode = IResizeMode.auto + }) + }} + /> */} + +
+ +
+ { + views[i].filters &&
+

filters

+ {views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} +
+ } +
) + } +
+ = specs.length} + style={{ marginTop: '5px' }} + text={intl.get('discovery.main.loadMore')} + onClick={() => { + setRenderAmount(a => a + RENDER_BATCH_SIZE) + }} + /> +
+
+} + +export default observer(PatternPage); \ No newline at end of file diff --git a/packages/rath-client/src/pages/semiAutomation/settings.tsx b/packages/rath-client/src/pages/semiAutomation/settings.tsx index b80029e6..594f417b 100644 --- a/packages/rath-client/src/pages/semiAutomation/settings.tsx +++ b/packages/rath-client/src/pages/semiAutomation/settings.tsx @@ -1,5 +1,5 @@ import { observer } from 'mobx-react-lite'; -import { ChoiceGroup, IChoiceGroupOption, Panel } from 'office-ui-fabric-react'; +import { ChoiceGroup, IChoiceGroupOption, Label, Panel, Toggle } from 'office-ui-fabric-react'; import React, { useMemo } from 'react'; import { useGlobalStore } from '../../store'; import intl from 'react-intl-universal'; @@ -13,7 +13,7 @@ const PatternSetting: React.FC = () => { { text: intl.get('discovery.main.vizsys.strict'), key: 'strict' } ] }, []) - const { showSettings, settings } = discoveryMainStore; + const { showSettings, settings, autoAsso } = discoveryMainStore; const { vizAlgo } = settings; return { selectedKey={vizAlgo} options={options} /> +
+ + { + discoveryMainStore.updateAutoAssoConfig('featViews', Boolean(checked)) + }} /> + { + discoveryMainStore.updateAutoAssoConfig('pattViews', Boolean(checked)) + }} /> + { + discoveryMainStore.updateAutoAssoConfig('filterViews', Boolean(checked)) + }} />
diff --git a/packages/rath-client/src/store/discovery/localTypes.ts b/packages/rath-client/src/store/discovery/localTypes.ts new file mode 100644 index 00000000..e75d76b4 --- /dev/null +++ b/packages/rath-client/src/store/discovery/localTypes.ts @@ -0,0 +1,32 @@ +import { IPattern } from "../../dev"; +import { IResizeMode } from "../../interfaces"; + +export interface ISetting { + vizAlgo: 'lite' | 'strict' +} +export interface IMainVizSetting { + interactive: boolean; + debug: boolean; + resize: { + mode: IResizeMode; + width: number; + height: number; + }; + nlg: boolean; +} + +export type IRenderViewKey = 'pattViews' | 'featViews' | 'filterViews'; + +export interface IAssoViews { + views: IPattern[]; + amount: number; +} + +export const RENDER_BATCH_SIZE = 5; + +export function makeInitAssoViews(initRenderAmount: number = 5): IAssoViews { + return { + views: [], + amount: initRenderAmount + } +} \ No newline at end of file diff --git a/packages/rath-client/src/store/discovery/mainStore.ts b/packages/rath-client/src/store/discovery/mainStore.ts index f72f9453..f3f8db2c 100644 --- a/packages/rath-client/src/store/discovery/mainStore.ts +++ b/packages/rath-client/src/store/discovery/mainStore.ts @@ -1,24 +1,36 @@ -import { makeAutoObservable, runInAction } from "mobx"; -import { IResizeMode } from "../../interfaces"; +import produce from "immer"; +import { makeAutoObservable, observable, runInAction } from "mobx"; +import { IPattern } from "../../dev"; +import { IFieldMeta, IResizeMode } from "../../interfaces"; +import { distVis } from "../../queries/distVis"; +import { labDistVis } from "../../queries/labdistVis"; +import { footmanEngineService } from "../../service"; +import { DataSourceStore } from "../dataSourceStore"; +import { IAssoViews, IMainVizSetting, IRenderViewKey, ISetting, makeInitAssoViews } from "./localTypes"; + +const RENDER_BATCH_SIZE = 5; -interface ISetting { - vizAlgo: 'lite' | 'strict' -} -interface IMainVizSetting { - interactive: boolean; - debug: boolean; - resize: { - mode: IResizeMode; - width: number; - height: number; - }; - nlg: boolean; -} export class DiscoveryMainStore { public settings: ISetting; public showSettings: boolean = false; public mainVizSetting: IMainVizSetting; - constructor () { + public pattViews: IAssoViews; + public featViews: IAssoViews; + public filterViews: IAssoViews; + private dataSourceStore: DataSourceStore; + public computing: boolean = false; + public mainView: IPattern | null = null; + public compareView: IPattern | null = null; + public showMiniFloatView: boolean = false; + public autoAsso: { + [key in IRenderViewKey]: boolean; + } = { + pattViews: true, + featViews: true, + filterViews: true + } + constructor (dataSourceStore: DataSourceStore) { + this.dataSourceStore = dataSourceStore; this.mainVizSetting = { interactive: false, debug: false, @@ -32,7 +44,17 @@ export class DiscoveryMainStore { this.settings = { vizAlgo: 'lite' } - makeAutoObservable(this); + this.pattViews = makeInitAssoViews(RENDER_BATCH_SIZE); + this.featViews = makeInitAssoViews(RENDER_BATCH_SIZE); + this.filterViews = makeInitAssoViews(RENDER_BATCH_SIZE); + + makeAutoObservable(this, { + pattViews: observable.shallow, + featViews: observable.shallow, + filterViews: observable.shallow, + mainView: observable.ref, + compareView: observable.ref + }); } public setShowSettings (show: boolean) { this.showSettings = show; @@ -40,9 +62,222 @@ export class DiscoveryMainStore { public updateSettings (skey: keyof ISetting, value: any) { this.settings[skey] = value; } + public updateAutoAssoConfig (akey: IRenderViewKey, value: boolean) { + this.autoAsso[akey] = value; + } public updateMainVizSettings (updater: (s: IMainVizSetting) => void) { runInAction(() => { updater(this.mainVizSetting); }) } + public setShowMiniFloatView (show: boolean) { + this.showMiniFloatView = show; + } + public get dataSource () { + return this.dataSourceStore.cleanedData; + } + public get fieldMetas () { + return this.dataSourceStore.fieldMetas; + } + public get pattSpecList () { + // TODO: 这里的设计并不会带来性能上的优化,过去只会被计算一次的整体,现在反而要算的更多。 + // 仅仅对于联想视图比较多的场景会有优化。 + const { amount, views } = this.pattViews; + const renderedViews = views.slice(0, amount); + if (this.settings.vizAlgo === 'lite') { + return renderedViews.map(v => distVis({ + pattern: v + })) + } else { + return renderedViews.map(v => labDistVis({ + pattern: v, + dataSource: this.dataSource + })) + } + } + public get featSpecList () { + // TODO: 这里的设计并不会带来性能上的优化,过去只会被计算一次的整体,现在反而要算的更多。 + // 仅仅对于联想视图比较多的场景会有优化。 + const { amount, views } = this.featViews; + const renderedViews = views.slice(0, amount); + if (this.settings.vizAlgo === 'lite') { + return renderedViews.map(v => distVis({ + pattern: v + })) + } else { + return renderedViews.map(v => labDistVis({ + pattern: v, + dataSource: this.dataSource + })) + } + } + public get filterSpecList () { + // TODO: 这里的设计并不会带来性能上的优化,过去只会被计算一次的整体,现在反而要算的更多。 + // 仅仅对于联想视图比较多的场景会有优化。 + const { amount, views } = this.filterViews; + const renderedViews = views.slice(0, amount); + if (this.settings.vizAlgo === 'lite') { + return renderedViews.map(v => distVis({ + pattern: v + })) + } else { + return renderedViews.map(v => labDistVis({ + pattern: v, + dataSource: this.dataSource + })) + } + } + + public changeRenderAmount (stateKey: IRenderViewKey, size: number) { + this[stateKey].amount = size; + } + // 为一个关联模块的渲染数量增加一个系统默认值 + public increaseRenderAmount (stateKey: IRenderViewKey) { + const safeSize = Math.min(this[stateKey].amount + RENDER_BATCH_SIZE, this[stateKey].views.length) + this.changeRenderAmount(stateKey, safeSize) + } + public async featAssociate () { + this.computing = true; + const { fieldMetas, dataSource, mainView } = this; + try { + const res = await footmanEngineService({ + dataSource, + fields: fieldMetas, + task: 'featureSelection', + props: mainView + }, 'local') + runInAction(() => { + this.featViews.views = res; + this.featViews.amount = RENDER_BATCH_SIZE; + this.computing = false; + }) + } catch (error) { + console.error(error); + this.computing = false; + } + } + public async pattAssociate () { + this.computing = true; + const { fieldMetas, dataSource, mainView } = this; + try { + const res = await footmanEngineService({ + dataSource, + fields: fieldMetas, + task: 'patterns', + props: mainView + }, 'local') + runInAction(() => { + this.pattViews.views = res; + this.pattViews.amount = RENDER_BATCH_SIZE; + this.computing = false; + }) + } catch (error) { + console.error(error); + this.computing = false; + } + } + public async initAssociate () { + this.computing = false; + const { dataSource, fieldMetas } = this; + try { + const res = await footmanEngineService({ + dataSource, + fields: fieldMetas, + task: 'univar' + }, 'local') + runInAction(() => { + this.computing = false; + this.pattViews.views = res; + this.pattViews.amount = RENDER_BATCH_SIZE; + }) + } catch (error) { + console.error(error); + this.computing = false; + } + } + public async filterAssociate () { + if (this.mainView === null) return; + this.computing = true; + const { fieldMetas, dataSource, mainView } = this; + try { + const res = await footmanEngineService({ + dataSource, + fields: fieldMetas, + task: 'filterSelection', + props: mainView + }, 'local') + runInAction(() => { + this.filterViews.views = res; + this.filterViews.amount = RENDER_BATCH_SIZE; + this.computing = false; + }) + } catch (error) { + console.error(error); + this.computing = false; + } + } + public removeMainViewFilter (filterFieldId: string) { + if (!this.mainView?.filters) return; + this.mainView = produce(this.mainView, draft => { + draft.filters = draft.filters!.filter(f => f.field.fid !== filterFieldId) + }) + } + public removeMainViewField (fieldId: string) { + if (this.mainView === null) return; + const targetFieldIndex = this.mainView.fields.findIndex(f => f.fid === fieldId); + this.mainView = produce(this.mainView, draft => { + draft.fields.splice(targetFieldIndex, 1) + }) + console.log(this.mainView) + } + public clearViews () { + this.featViews = makeInitAssoViews(); + this.pattViews = makeInitAssoViews(); + this.filterViews = makeInitAssoViews(); + } + public initRenderSize () { + this.featViews.amount = RENDER_BATCH_SIZE; + this.pattViews.amount = RENDER_BATCH_SIZE; + this.filterViews.amount = RENDER_BATCH_SIZE; + } + public updateMainView (view: IPattern) { + this.mainView = view; + this.initAssociate() + } + public updateCompareView (view: IPattern) { + this.compareView = view; + this.mainVizSetting.resize.mode = IResizeMode.auto; + } + public async explainViewDiff (view1: IPattern, view2: IPattern) { + if (this.mainView === null) return; + this.computing = true; + const { fieldMetas, dataSource } = this; + try { + const res = await footmanEngineService<{ features: IFieldMeta[] }>({ + dataSource, + fields: fieldMetas, + task: 'filterSelection', + props: [view1, view2] + }, 'local') + runInAction(() => { + if (this.mainView) { + this.clearViews(); + this.featViews.views = [ + { + ...this.mainView, + fields: [...this.mainView!.fields, ...res.features] + }, + { + ...this.mainView, + fields: [...this.mainView!.fields, ...res.features] + } + ] + } + this.computing = false; + }) + } catch (error) { + console.error(error); + this.computing = false; + } + } } \ No newline at end of file diff --git a/packages/rath-client/src/store/index.tsx b/packages/rath-client/src/store/index.tsx index 83823f71..b8dd41b5 100644 --- a/packages/rath-client/src/store/index.tsx +++ b/packages/rath-client/src/store/index.tsx @@ -35,7 +35,7 @@ const galleryStore = new GalleryStore(litePipeStore); const noteBookStore = new NoteBookStore(litePipeStore); const dashBoardStore = new DashBoardStore(litePipeStore); const exploreStore = new ExploreStore(ltsPipeLineStore); -const discoveryMainStore = new DiscoveryMainStore(); +const discoveryMainStore = new DiscoveryMainStore(dataSourceStore); const storeCol: StoreCollection = { diff --git a/packages/rath-client/src/utils/index.ts b/packages/rath-client/src/utils/index.ts index 9b3ee028..c61c1936 100644 --- a/packages/rath-client/src/utils/index.ts +++ b/packages/rath-client/src/utils/index.ts @@ -101,6 +101,18 @@ export function findRathSafeColumnIndex (fields: T[]): return fields.findIndex(f => f.fid === RATH_INDEX_COLUMN_KEY); } +export function throttle (func: F, delay: number) { + let timer: number | null = null; + return function () { + if (timer === null) { + timer = window.setTimeout(() => { + func(); + timer = null; + }, delay); + } + } +} + export { isASCII, inferAnalyticType, From 29100d155a52548ce7c429b4ca530421e3f2cede Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Fri, 15 Jul 2022 09:56:35 +0800 Subject: [PATCH 02/15] fix: auto asso bugs --- .../rath-client/public/locales/en-US.json | 11 +- .../rath-client/public/locales/zh-CN.json | 11 +- packages/rath-client/src/App.css | 8 + .../src/pages/semiAutomation/components.tsx | 7 +- .../pages/semiAutomation/focusZone/index.tsx | 39 +-- .../focusZone/miniFloatCanvas.tsx | 4 +- .../src/pages/semiAutomation/index.tsx | 134 +------- .../predictZone/featSegment.tsx | 32 +- .../predictZone/filterSegment.tsx | 29 +- .../predictZone/pattSegment.tsx | 55 ++- .../pages/semiAutomation/predictZone/tmp.txt | 313 ------------------ .../src/pages/semiAutomation/utils.ts | 22 +- .../src/store/discovery/mainStore.ts | 6 +- 13 files changed, 127 insertions(+), 544 deletions(-) delete mode 100644 packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index a8609c4c..a397aa51 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -5,7 +5,9 @@ "name": "Name", "home": "Home", "settings": "Settings", - "history": "History" + "history": "History", + "hide": "Hide", + "expand": "Expand" }, "menu": { "dataSource": "DataSource", @@ -286,8 +288,8 @@ "discovery": { "main": { "mainView": "Main View", - "relatePatterns": "relate patterns(default)", - "relateFeatures": "relate features", + "relatePatterns": "associate patterns", + "relateFeatures": "associate features", "explainDiff": "explain diff", "pointInterests": "split", "pin": "pin", @@ -304,5 +306,8 @@ "filters": "Associated Subsets" } } + }, + "desc": { + "dataViewInfo": "Pattern in {measures} by {dimensions} where {filters}." } } \ No newline at end of file diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 1399cc58..e09cb643 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -5,7 +5,9 @@ "name": "名称", "home": "主页", "settings": "设置", - "history": "历史记录" + "history": "历史记录", + "hide": "隐藏", + "expand": "展开" }, "menu": { "dataSource": "数据源", @@ -286,8 +288,8 @@ "discovery": { "main": { "mainView": "主视图", - "relatePatterns": "关联规律(默认)", - "relateFeatures": "特征推荐", + "relatePatterns": "关联规律", + "relateFeatures": "关联特征", "explainDiff": "差异性分析", "pointInterests": "拆分", "pin": "深入分析", @@ -304,5 +306,8 @@ "filters": "关联子集" } } + }, + "desc": { + "dataViewInfo": "在 {dimensions} 中 {measures} 的表现, 限制条件 {filters}." } } \ No newline at end of file diff --git a/packages/rath-client/src/App.css b/packages/rath-client/src/App.css index ece34a40..7167fac6 100644 --- a/packages/rath-client/src/App.css +++ b/packages/rath-client/src/App.css @@ -54,6 +54,14 @@ h2, h3 { font-size: 12px; color: #8a8886; } + +.pure-card{ + background-color: #fff; + padding: 28px; + border-radius: 2px; + box-shadow: 0 1.6px 3.6px 0 rgba(0,0,0,0.132), 0 0.3px 0.9px 0 rgba(0,0,0,0.108); + margin-bottom: 1em; +} .card{ background-color: #fff; padding: 28px; diff --git a/packages/rath-client/src/pages/semiAutomation/components.tsx b/packages/rath-client/src/pages/semiAutomation/components.tsx index a10d7f93..f09ca6df 100644 --- a/packages/rath-client/src/pages/semiAutomation/components.tsx +++ b/packages/rath-client/src/pages/semiAutomation/components.tsx @@ -16,7 +16,10 @@ export const AssoContainer = styled.div` padding: 10px; position: relative; .chart-container{ - min-height: 300px; + min-height: 280px; + } + .chart-desc{ + font-size: 12px; } } ` @@ -40,7 +43,7 @@ export const MainViewContainer = styled.div` } .fields-container{ display: flex; - padding: 1em 0em; + padding: 1em 0em 0em 0em; } .action-buttons{ margin: 6px 0px; diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx index 8cfc0544..60952944 100644 --- a/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/index.tsx @@ -1,5 +1,5 @@ import { observer } from 'mobx-react-lite'; -import { DefaultButton, PrimaryButton } from 'office-ui-fabric-react'; +import { PrimaryButton } from 'office-ui-fabric-react'; import React, { useCallback } from 'react'; import { IFieldMeta } from '../../../interfaces'; import { useGlobalStore } from '../../../store'; @@ -9,21 +9,12 @@ import MainCanvas from './mainCanvas'; import intl from 'react-intl-universal'; import MiniFloatCanvas from './miniFloatCanvas'; -const BUTTON_STYLE = { marginRight: '1em' } +const BUTTON_STYLE = { marginRight: '1em', marginTop: '1em' } const FocusZone: React.FC = props => { const { discoveryMainStore } = useGlobalStore(); - const { mainView, compareView, showMiniFloatView, autoAsso } = discoveryMainStore; - const advicePureFeature = useCallback(() => { - discoveryMainStore.featAssociate() - }, [discoveryMainStore]) - const assViews = useCallback(() => { - discoveryMainStore.pattAssociate(); - discoveryMainStore.featAssociate(); - }, [discoveryMainStore]) - const recommandFilter = useCallback(() => { - discoveryMainStore.filterAssociate(); - }, [discoveryMainStore]) + const { mainView, compareView, showMiniFloatView } = discoveryMainStore; + const explainDiff = useCallback(() => { if (mainView && compareView) { discoveryMainStore.explainViewDiff(mainView, compareView); @@ -39,6 +30,7 @@ const FocusZone: React.FC = props => { {compareView && }
+
{ mainView && mainView.fields.map((f: IFieldMeta) => { }
- { - !autoAsso.pattViews && - } - { - !autoAsso.featViews && - } - { - !autoAsso.filterViews && - }
} diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx index d84bb288..74760e7a 100644 --- a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx @@ -2,6 +2,7 @@ import { observer } from 'mobx-react-lite'; import { CommandBarButton, IconButton } from 'office-ui-fabric-react'; import React, { useMemo, useState } from 'react'; import styled from 'styled-components'; +import intl from 'react-intl-universal' import ReactVega from '../../../components/react-vega'; import { IPattern } from '../../../dev'; @@ -52,7 +53,7 @@ const MiniFloatCanvas: React.FC = props => { { !hide && { setHide(v => !v) }} @@ -61,6 +62,7 @@ const MiniFloatCanvas: React.FC = props => { { hide && { setHide(v => !v) }} diff --git a/packages/rath-client/src/pages/semiAutomation/index.tsx b/packages/rath-client/src/pages/semiAutomation/index.tsx index 9b5da3f3..0273147e 100644 --- a/packages/rath-client/src/pages/semiAutomation/index.tsx +++ b/packages/rath-client/src/pages/semiAutomation/index.tsx @@ -1,149 +1,31 @@ import { observer } from 'mobx-react-lite'; -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { IFilter, IPattern } from '../../dev'; +import React, { useEffect, useRef } from 'react'; import { useGlobalStore } from '../../store'; -import { distVis } from '../../queries/distVis' -import ReactVega from '../../components/react-vega'; -import { DefaultButton, Stack, PrimaryButton, ActionButton, CommandButton, Spinner } from 'office-ui-fabric-react'; -import { AssoContainer, LoadingLayer, MainViewContainer } from './components'; -import ViewField from '../megaAutomation/vizOperation/viewField'; -import { IFieldMeta, IResizeMode, IRow } from '../../interfaces'; -import { footmanEngineService } from '../../service'; +import { ActionButton } from 'office-ui-fabric-react'; import Settings from './settings'; -import { labDistVis } from '../../queries/labdistVis'; -import { notify } from '../../components/error'; import intl from 'react-intl-universal'; -import MainCanvas from './mainCanvas'; import FocusZone from './focusZone'; import PredictZone from './predictZone'; import { throttle } from '../../utils'; -const BUTTON_STYLE = { marginRight: '1em' } - -function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { - if (!filters || filters.length === 0) return dataSource; - const subset: IRow[] = []; - const filterValueSetList: Array> = []; - for (let i = 0; i < filters.length; i++) { - filterValueSetList.push(new Set(filters[i].values)) - } - for (let i = 0; i < dataSource.length; i++) { - for (let j = 0; j < filters.length; j++) { - if (filterValueSetList[j].has(dataSource[i][filters[j].field.fid])) { - subset.push(dataSource[i]) - } - } - } - return subset - // return dataSource.filter(row => { - // return filters.every(f => f.values.includes(row[f.field.fid])) - // }) -} - -const RENDER_BATCH_SIZE = 5; const PatternPage: React.FC = () => { const focusZoneContainer = useRef(null); - const { dataSourceStore, discoveryMainStore } = useGlobalStore(); - // const { fieldMetas, cleanedData } = dataSourceStore; - const [views, setViews] = useState([]); - const [pined, setPined] = useState(null); - const [computing, setComputing] = useState(false); - const [renderAmount, setRenderAmount] = useState(RENDER_BATCH_SIZE); - const [mergeView, setMergeView] = useState(null); - const { mainVizSetting, - featViews, - filterViews, - pattViews, + const { discoveryMainStore } = useGlobalStore(); + const { fieldMetas, dataSource, - featSpecList, - pattSpecList, - filterSpecList } = discoveryMainStore; - const vizRecSys = discoveryMainStore.settings.vizAlgo; useEffect(() => { + if (dataSource.length > 1e5) { + discoveryMainStore.updateSettings('vizAlgo', 'lite') + } discoveryMainStore.initAssociate(); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [fieldMetas, dataSource]) - const assViews = useCallback((view: IPattern) => { - discoveryMainStore.pattAssociate(); - discoveryMainStore.featAssociate(); - }, []) - - const adviceCompareFeature = useCallback(() => { - if (pined === null || mergeView === null) return; - // footmanEngineService({ - // dataSource: cleanedData, - // fields: fieldMetas, - // task: 'featureSelection', - // props: - // }) - footmanEngineService({ - dataSource, - fields: fieldMetas, - task: 'comparison', - props: [pined, mergeView] - }).then(res => { - if (res !== null) { - setViews([ - { - ...pined, - fields: [...pined.fields, ...res.features] - }, - { - ...mergeView, - fields: [...mergeView.fields, ...res.features] - }, - ]) - } - }).catch(err => { - notify({ - title: 'comparsion error', - type: 'error', - content: `${err}` - }) - }) - }, [pined, mergeView, dataSource, fieldMetas]) - - const advicePureFeature = useCallback(() => { - discoveryMainStore.featAssociate() - }, []) - - const removeFromPined = useCallback((fid: string) => { - if (pined === null) return; - const fields: IFieldMeta[] = [...pined.fields]; - const targetIndex = fields.findIndex(f => f.fid === fid); - if (targetIndex > -1) { - fields.splice(targetIndex, 1) - setPined({ - ...pined, - fields: fields - }) - } - }, [pined]) - - const recommandFilter = useCallback(() => { - discoveryMainStore.filterAssociate(); - }, []) - const removePinedFilter = useCallback((filterField: IFieldMeta) => { - setPined(p => { - if (!p?.filters) return p; - return { - ...p, - filters: p.filters.filter(f => f.field.fid !== filterField.fid) - } - }) - }, []) - - useEffect(() => { - if (pined) { - assViews(pined) - setRenderAmount(RENDER_BATCH_SIZE) - } - }, [pined, assViews]) useEffect(() => { const ele = document.querySelector('.main-app-content'); diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx index 30072a1e..3a4ad794 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx @@ -1,7 +1,7 @@ import React, { useCallback } from 'react'; import { observer } from 'mobx-react-lite'; import intl from 'react-intl-universal'; -import { CommandButton, DefaultButton, Spinner, Stack } from 'office-ui-fabric-react'; +import { CommandButton, DefaultButton, PrimaryButton, Spinner, Stack } from 'office-ui-fabric-react'; import { useGlobalStore } from '../../../store'; import { AssoContainer, LoadingLayer } from '../components'; @@ -10,13 +10,24 @@ import { applyFilter } from '../utils'; const FeatSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); - const { featSpecList, computing, featViews, mainVizSetting, dataSource } = discoveryMainStore; + const { featSpecList, computing, featViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; const loadMore = useCallback(() => { discoveryMainStore.increaseRenderAmount('featViews'); }, [discoveryMainStore]) - if (featViews.views.length === 0) return
- return
-

{intl.get('discovery.main.associate.features')}

+ const advicePureFeature = useCallback(() => { + discoveryMainStore.featAssociate() + }, [discoveryMainStore]) + if (featViews.views.length === 0 && autoAsso.featViews) return
+ return
+

{intl.get('discovery.main.associate.features')}

+ { + !autoAsso.featViews && + } { featSpecList.map((spec, i) =>
@@ -41,12 +52,11 @@ const FeatSegment: React.FC = () => { dataSource={applyFilter(dataSource, featViews.views[i].filters)} />
- { - featViews.views[i].filters &&
-

filters

- {featViews.views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} -
- } +
+ { featViews.views[i].fields?.filter(f => f.analyticType === 'dimension').map(f => f.name || f.fid).join(', ') }
+ { featViews.views[i].fields?.filter(f => f.analyticType === 'measure').map(f => f.name || f.fid).join(', ') }
+ { featViews.views[i].filters?.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n') } +
) } diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx index 315d8dcb..43fac318 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx @@ -10,13 +10,23 @@ import { applyFilter } from '../utils'; const FilterSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); - const { filterSpecList, computing, filterViews, mainVizSetting, dataSource } = discoveryMainStore; + const { filterSpecList, computing, filterViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; const loadMore = useCallback(() => { discoveryMainStore.increaseRenderAmount('featViews'); }, [discoveryMainStore]) - if (filterViews.views.length === 0) return
- return
-

{intl.get('discovery.main.associate.filters')}

+ const recommandFilter = useCallback(() => { + discoveryMainStore.filterAssociate(); + }, [discoveryMainStore]) + if (filterViews.views.length === 0 && autoAsso.filterViews) return
+ return
+

{intl.get('discovery.main.associate.filters')}

+ { + !autoAsso.filterViews && + } { filterSpecList.map((spec, i) =>
@@ -48,12 +58,11 @@ const FilterSegment: React.FC = () => { dataSource={applyFilter(dataSource, filterViews.views[i].filters)} />
- { - filterViews.views[i].filters &&
-

filters

- {filterViews.views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} -
- } +
+ { filterViews.views[i].fields?.filter(f => f.analyticType === 'dimension').map(f => f.name || f.fid).join(', ') }
+ { filterViews.views[i].fields?.filter(f => f.analyticType === 'measure').map(f => f.name || f.fid).join(', ') }
+ { filterViews.views[i].filters?.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n') } +
) } diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx index a1b4a0e4..ca321e8a 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx @@ -10,13 +10,23 @@ import { applyFilter } from '../utils'; const PattSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); - const { pattSpecList, computing, pattViews, mainVizSetting, dataSource } = discoveryMainStore; + const { pattSpecList, computing, pattViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; const loadMore = useCallback(() => { discoveryMainStore.increaseRenderAmount('featViews'); }, [discoveryMainStore]) - if (pattViews.views.length === 0) return
- return
-

{intl.get('discovery.main.associate.patterns')}

+ const assViews = useCallback(() => { + discoveryMainStore.pattAssociate(); + }, [discoveryMainStore]) + if (pattViews.views.length === 0 && autoAsso.pattViews) return
+ return
+

{intl.get('discovery.main.associate.patterns')}

+ { + !autoAsso.pattViews && + } { pattSpecList.map((spec, i) =>
@@ -33,32 +43,6 @@ const PattSegment: React.FC = () => { discoveryMainStore.updateMainView(pattViews.views[i]) }} /> - {/* { - discoveryMainStore.updateMainView(pattViews.views[i]) - setMergeView(views[i]) - discoveryMainStore.updateMainVizSettings(s => { - s.resize.mode = IResizeMode.auto - }) - }} - /> */} - {/* { - setPined(views[i]) - }} - /> - { - setMergeView(views[i]) - discoveryMainStore.updateMainVizSettings(s => { - s.resize.mode = IResizeMode.auto - }) - }} - /> */}
{ dataSource={applyFilter(dataSource, pattViews.views[i].filters)} />
- { - pattViews.views[i].filters &&
-

filters

- {pattViews.views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} -
- } +
+ { pattViews.views[i].fields?.filter(f => f.analyticType === 'dimension').map(f => f.name || f.fid).join(', ') }
+ { pattViews.views[i].fields?.filter(f => f.analyticType === 'measure').map(f => f.name || f.fid).join(', ') }
+ { pattViews.views[i].filters?.map(f => `${f.field.name || f.field.fid} = ${f.values.join(', ')}`).join('\n') } +
) }
diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt b/packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt deleted file mode 100644 index 2391aa41..00000000 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/tmp.txt +++ /dev/null @@ -1,313 +0,0 @@ -import { observer } from 'mobx-react-lite'; -import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import { IFilter, IPattern } from '../../dev'; -import { useGlobalStore } from '../../store'; -import { distVis } from '../../queries/distVis' -import ReactVega from '../../components/react-vega'; -import { DefaultButton, Stack, PrimaryButton, ActionButton, CommandButton, Spinner } from 'office-ui-fabric-react'; -import { AssoContainer, LoadingLayer, MainViewContainer } from './components'; -import ViewField from '../megaAutomation/vizOperation/viewField'; -import { IFieldMeta, IResizeMode, IRow } from '../../interfaces'; -import { footmanEngineService } from '../../service'; -import Settings from './settings'; -import { labDistVis } from '../../queries/labdistVis'; -import { notify } from '../../components/error'; -import intl from 'react-intl-universal'; -import MainCanvas from './mainCanvas'; - -const BUTTON_STYLE = { marginRight: '1em' } - -function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { - if (!filters || filters.length === 0) return dataSource; - const subset: IRow[] = []; - const filterValueSetList: Array> = []; - for (let i = 0; i < filters.length; i++) { - filterValueSetList.push(new Set(filters[i].values)) - } - for (let i = 0; i < dataSource.length; i++) { - for (let j = 0; j < filters.length; j++) { - if (filterValueSetList[j].has(dataSource[i][filters[j].field.fid])) { - subset.push(dataSource[i]) - } - } - } - return subset - // return dataSource.filter(row => { - // return filters.every(f => f.values.includes(row[f.field.fid])) - // }) -} - -const RENDER_BATCH_SIZE = 5; - -const PatternPage: React.FC = () => { - const { dataSourceStore, discoveryMainStore } = useGlobalStore(); - // const { fieldMetas, cleanedData } = dataSourceStore; - const [views, setViews] = useState([]); - const [pined, setPined] = useState(null); - const [computing, setComputing] = useState(false); - const [renderAmount, setRenderAmount] = useState(RENDER_BATCH_SIZE); - const [mergeView, setMergeView] = useState(null); - const { mainVizSetting, - featViews, - filterViews, - pattViews, - fieldMetas, - dataSource, - featSpecList, - pattSpecList, - filterSpecList - } = discoveryMainStore; - const vizRecSys = discoveryMainStore.settings.vizAlgo; - - useEffect(() => { - discoveryMainStore.initAssociate(); - }, [fieldMetas, dataSource]) - - const assViews = useCallback((view: IPattern) => { - discoveryMainStore.pattAssociate(); - discoveryMainStore.featAssociate(); - }, []) - - const adviceCompareFeature = useCallback(() => { - if (pined === null || mergeView === null) return; - // footmanEngineService({ - // dataSource: cleanedData, - // fields: fieldMetas, - // task: 'featureSelection', - // props: - // }) - footmanEngineService({ - dataSource, - fields: fieldMetas, - task: 'comparison', - props: [pined, mergeView] - }).then(res => { - if (res !== null) { - setViews([ - { - ...pined, - fields: [...pined.fields, ...res.features] - }, - { - ...mergeView, - fields: [...mergeView.fields, ...res.features] - }, - ]) - } - }).catch(err => { - notify({ - title: 'comparsion error', - type: 'error', - content: `${err}` - }) - }) - }, [pined, mergeView, dataSource, fieldMetas]) - - const advicePureFeature = useCallback(() => { - discoveryMainStore.featAssociate() - }, []) - - const removeFromPined = useCallback((fid: string) => { - if (pined === null) return; - const fields: IFieldMeta[] = [...pined.fields]; - const targetIndex = fields.findIndex(f => f.fid === fid); - if (targetIndex > -1) { - fields.splice(targetIndex, 1) - setPined({ - ...pined, - fields: fields - }) - } - }, [pined]) - - const recommandFilter = useCallback(() => { - discoveryMainStore.filterAssociate(); - }, []) - - const removePinedFilter = useCallback((filterField: IFieldMeta) => { - setPined(p => { - if (!p?.filters) return p; - return { - ...p, - filters: p.filters.filter(f => f.field.fid !== filterField.fid) - } - }) - }, []) - - useEffect(() => { - if (pined) { - assViews(pined) - setRenderAmount(RENDER_BATCH_SIZE) - } - }, [pined, assViews]) - - return
- -
- { - discoveryMainStore.setShowSettings(true); - }} - /> - -

{intl.get('discovery.main.mainView')}

-
-
- { - pined !== null && - } -
- - {mergeView !== null &&
- {/* { - vizRecSys === 'lite' && - } */} - {/* { - vizRecSys === 'strict' && - } */} -
} - -
-
- { - ma && ma.fields.map((f: IFieldMeta) => { removeFromPined(f.fid) }} - />) - } -
-
- { - pined && pined.filters && pined.filters.map(f => { - removePinedFilter(f.field) - }} - />) - } -
-
- { - if (pined) { - assViews(pined) - } - }} - /> - - - -
-
-
- - { - specs.slice(0, renderAmount).map((spec, i) =>
- { - computing && - - - } - - { - setPined(views[i]) - }} - /> - { - setMergeView(views[i]) - discoveryMainStore.updateMainVizSettings(s => { - s.resize.mode = IResizeMode.auto - }) - }} - /> - {/* { - setPined(views[i]) - }} - /> - { - setMergeView(views[i]) - discoveryMainStore.updateMainVizSettings(s => { - s.resize.mode = IResizeMode.auto - }) - }} - /> */} - -
- -
- { - views[i].filters &&
-

filters

- {views[i].filters!.map(f => `${f.field.name || f.field.fid} = ${f.values.join(',')}`).join('\n')} -
- } -
) - } -
- = specs.length} - style={{ marginTop: '5px' }} - text={intl.get('discovery.main.loadMore')} - onClick={() => { - setRenderAmount(a => a + RENDER_BATCH_SIZE) - }} - /> -
-
-} - -export default observer(PatternPage); \ No newline at end of file diff --git a/packages/rath-client/src/pages/semiAutomation/utils.ts b/packages/rath-client/src/pages/semiAutomation/utils.ts index 1bc47329..9c820832 100644 --- a/packages/rath-client/src/pages/semiAutomation/utils.ts +++ b/packages/rath-client/src/pages/semiAutomation/utils.ts @@ -1,9 +1,25 @@ import { IFilter } from "../../dev"; import { IRow } from "../../interfaces"; +// export function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { +// if (!filters || filters.length === 0) return dataSource; +// return dataSource.filter(row => { +// return filters.every(f => f.values.includes(row[f.field.fid])) +// }) +// } export function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { if (!filters || filters.length === 0) return dataSource; - return dataSource.filter(row => { - return filters.every(f => f.values.includes(row[f.field.fid])) - }) + const subset: IRow[] = []; + const filterValueSetList: Array> = []; + for (let i = 0; i < filters.length; i++) { + filterValueSetList.push(new Set(filters[i].values)) + } + for (let i = 0; i < dataSource.length; i++) { + for (let j = 0; j < filters.length; j++) { + if (filterValueSetList[j].has(dataSource[i][filters[j].field.fid])) { + subset.push(dataSource[i]) + } + } + } + return subset } \ No newline at end of file diff --git a/packages/rath-client/src/store/discovery/mainStore.ts b/packages/rath-client/src/store/discovery/mainStore.ts index f3f8db2c..9c8111e5 100644 --- a/packages/rath-client/src/store/discovery/mainStore.ts +++ b/packages/rath-client/src/store/discovery/mainStore.ts @@ -42,7 +42,7 @@ export class DiscoveryMainStore { nlg: false } this.settings = { - vizAlgo: 'lite' + vizAlgo: 'strict' } this.pattViews = makeInitAssoViews(RENDER_BATCH_SIZE); this.featViews = makeInitAssoViews(RENDER_BATCH_SIZE); @@ -179,6 +179,7 @@ export class DiscoveryMainStore { public async initAssociate () { this.computing = false; const { dataSource, fieldMetas } = this; + console.log('init') try { const res = await footmanEngineService({ dataSource, @@ -242,7 +243,8 @@ export class DiscoveryMainStore { } public updateMainView (view: IPattern) { this.mainView = view; - this.initAssociate() + // this.initAssociate() + this.clearViews(); } public updateCompareView (view: IPattern) { this.compareView = view; From 8cef38ecfa664cd67609fe7844a7c0fc238b2a25 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Fri, 15 Jul 2022 10:00:13 +0800 Subject: [PATCH 03/15] fix: channel priority --- packages/rath-client/src/queries/labdistVis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rath-client/src/queries/labdistVis.ts b/packages/rath-client/src/queries/labdistVis.ts index ca7386f0..b547316c 100644 --- a/packages/rath-client/src/queries/labdistVis.ts +++ b/packages/rath-client/src/queries/labdistVis.ts @@ -17,7 +17,7 @@ export const geomTypeMap: { [key: string]: any } = { }; const channels = { - quantitative: ['y', 'x', 'size', 'opacity', 'color'], + quantitative: ['y', 'x', 'size', 'color', 'opacity'], ordinal: ['y', 'x', 'opacity', 'color', 'size', 'shape'], nominal: ['y', 'x', 'color', 'row', 'column', 'opacity', 'size', 'shape'], temporal: ['y', 'x', 'size', 'color', 'opacity', 'shape'] From 56cea781614cf5fe9359e0f2936aede319dec1be Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Fri, 15 Jul 2022 10:13:59 +0800 Subject: [PATCH 04/15] fix: clear vega when unmount --- packages/rath-client/src/components/radarChart.tsx | 9 ++++++++- .../dataSource/dataTable/distributionMiniChart.tsx | 9 ++++++++- .../src/pages/dataSource/metaView/distChart.tsx | 10 +++++++++- packages/rath-client/src/visBuilder/vegaBase.tsx | 9 ++++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/rath-client/src/components/radarChart.tsx b/packages/rath-client/src/components/radarChart.tsx index c273e37a..dd0d4ba4 100644 --- a/packages/rath-client/src/components/radarChart.tsx +++ b/packages/rath-client/src/components/radarChart.tsx @@ -47,7 +47,7 @@ const RadarChart: React.FC = props => { }, [keyField, valueField, dataSource, threshold]) useEffect(() => { if (container.current) { - embed(container.current, { + const resultPromise = embed(container.current, { width: 280, height: 280, padding: 50, @@ -227,6 +227,13 @@ const RadarChart: React.FC = props => { } as any, { actions: false }); + return () => { + resultPromise.then(res => { + if (res) { + res.finalize() + } + }).catch(console.error) + } } }, [viewData]); return
diff --git a/packages/rath-client/src/pages/dataSource/dataTable/distributionMiniChart.tsx b/packages/rath-client/src/pages/dataSource/dataTable/distributionMiniChart.tsx index 642ca2f9..1cb582c8 100644 --- a/packages/rath-client/src/pages/dataSource/dataTable/distributionMiniChart.tsx +++ b/packages/rath-client/src/pages/dataSource/dataTable/distributionMiniChart.tsx @@ -38,7 +38,7 @@ const DistributionChart: React.FC = (props) => { } else if (fieldType === 'ordinal' && hasIndex) { sortBy = { field: 'index' } } - embed(chart.current, { + const resultPromise = embed(chart.current, { background: 'rgba(0,0,0,0)', data: { values @@ -62,6 +62,13 @@ const DistributionChart: React.FC = (props) => { }, { actions: false }) + return () => { + resultPromise.then(res => { + if (res) { + res.finalize() + } + }).catch(console.error) + } } }, [x, y, dataSource, fieldType]) return
diff --git a/packages/rath-client/src/pages/dataSource/metaView/distChart.tsx b/packages/rath-client/src/pages/dataSource/metaView/distChart.tsx index fe086a65..d44c4c84 100644 --- a/packages/rath-client/src/pages/dataSource/metaView/distChart.tsx +++ b/packages/rath-client/src/pages/dataSource/metaView/distChart.tsx @@ -78,7 +78,7 @@ const DistributionChart: React.FC = (props) => { useEffect(() => { if (chart.current) { - embed(chart.current, { + const resultPromise = embed(chart.current, { background: 'rgba(0,0,0,0)', data: { name: DATA_NAME @@ -112,7 +112,15 @@ const DistributionChart: React.FC = (props) => { actions: false }).then(res => { setView(res.view); + return res }) + return () => { + resultPromise.then(res => { + if (res) { + res.finalize() + } + }).catch(console.error) + } } }, [x, y, sortBy, semanticType]) useEffect(() => { diff --git a/packages/rath-client/src/visBuilder/vegaBase.tsx b/packages/rath-client/src/visBuilder/vegaBase.tsx index 8ab7e6c8..7c421e0b 100644 --- a/packages/rath-client/src/visBuilder/vegaBase.tsx +++ b/packages/rath-client/src/visBuilder/vegaBase.tsx @@ -107,13 +107,20 @@ const BaseChart: React.FC = (props) => { } let spec = mode === 'dist' ? baseVis(params) : commonVis(params); globalRef.baseVisSpec = spec; - embed(container.current, spec, { + const resultPromise = embed(container.current, spec, { mode: 'vega-lite', editorUrl: EDITOR_URL, actions: debug }).catch(err => { console.error('[VIS ERROR]', err) }) + return () => { + resultPromise.then(res => { + if (res) { + res.finalize() + } + }).catch(console.error) + } } } }, [schema, From c02589a1c32b24a3e96d162b607318968e9b355b Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Tue, 26 Jul 2022 16:27:14 +0800 Subject: [PATCH 05/15] feat: use new ver of vi --- package.json | 3 +- packages/graphic-walker/package.json | 2 +- packages/rath-client/config-overrides.js | 17 +++- packages/rath-client/package.json | 5 +- .../rath-client/src/components/radarChart.tsx | 2 +- packages/rath-client/src/dev/extend.ts | 6 ++ packages/rath-client/src/dev/index.ts | 5 +- packages/rath-client/src/dev/utils.ts | 9 +- .../pages/dataSource/dataTable/headerCell.tsx | 2 +- packages/rath-client/src/pages/dev/index.tsx | 2 +- .../predictZone/filterSegment.tsx | 2 +- .../predictZone/pattSegment.tsx | 2 +- packages/rath-client/src/queries/baseVis.ts | 2 +- packages/rath-client/src/queries/commonVis.ts | 2 +- .../rath-client/src/queries/featureVis.ts | 2 +- packages/rath-client/src/queries/index.ts | 2 +- .../rath-client/src/queries/labdistVis.ts | 4 +- packages/rath-client/src/queries/targetVis.ts | 2 +- packages/rath-client/src/queries/utils.ts | 2 +- packages/rath-client/src/service.ts | 2 +- .../rath-client/src/store/dataSourceStore.ts | 2 +- .../rath-client/src/store/exploreStore.ts | 3 +- .../src/store/pipeLineStore/lts.ts | 3 +- packages/rath-client/src/utils/transform.ts | 3 +- .../rath-client/src/visBuilder/vegaBase.tsx | 1 - .../rath-client/src/workers/dev.worker.js | 2 +- .../rath-client/src/workers/engine/cleaner.ts | 4 +- .../rath-client/src/workers/engine/core.ts | 88 +++++++++---------- .../src/workers/engine/index.worker.js | 5 ++ .../rath-client/src/workers/engine/service.ts | 8 +- .../src/workers/fieldsSummary.worker.js | 4 +- packages/rath-client/tsconfig.json | 7 +- yarn.lock | 28 +++--- 33 files changed, 132 insertions(+), 101 deletions(-) create mode 100644 packages/rath-client/src/dev/extend.ts diff --git a/package.json b/package.json index 068edb1f..24eeb947 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ "*/webpack-cli", "**/vega", "**/vega-lite", - "**/vega-embed" + "**/vega-embed", + "**/visual-insights" ] }, "homepage": "https://github.com/ObservedObserver/showme#readme", diff --git a/packages/graphic-walker/package.json b/packages/graphic-walker/package.json index 373d3be3..775d69de 100644 --- a/packages/graphic-walker/package.json +++ b/packages/graphic-walker/package.json @@ -2,7 +2,7 @@ "name": "@kanaries/graphic-walker", "version": "0.1.0", "scripts": { - "dev": "vite", + "dev": "vite --host", "build": "tsc && vite build", "serve": "vite preview", "type": "tsc src/lib.ts --declaration --emitDeclarationOnly --jsx react --esModuleInterop --outDir dist" diff --git a/packages/rath-client/config-overrides.js b/packages/rath-client/config-overrides.js index bf8afc71..bec69d3e 100644 --- a/packages/rath-client/config-overrides.js +++ b/packages/rath-client/config-overrides.js @@ -1,4 +1,14 @@ -module.exports = function override(config, env) { +const path = require('path'); +const { override, babelInclude } = require('customize-cra') + +// const mid_override = override( +// babelInclude([ +// path.resolve('src'), +// path.resolve(__dirname, '../../node_modules/visual-insights') +// ]) +// ) + +module.exports = function final_override(config, env) { // do stuff with the webpack config... config.module.rules.push({ test: /\.worker\.js$/, @@ -11,5 +21,8 @@ module.exports = function override(config, env) { }, }) config.output.globalObject = 'self' - return config; + config.module = config.module || {}; + config.module.unknownContextCritical = false + return config + // return mid_override(config) }; \ No newline at end of file diff --git a/packages/rath-client/package.json b/packages/rath-client/package.json index 5106cedd..a3336d3c 100644 --- a/packages/rath-client/package.json +++ b/packages/rath-client/package.json @@ -45,7 +45,7 @@ "vega": "^5.22.1", "vega-embed": "^6.20.8", "vega-lite": "^5.2.0", - "visual-insights": "0.8.11", + "visual-insights": "0.9.11", "web-vitals": "^0.2.4", "worker-loader": "^3.0.7" }, @@ -60,8 +60,9 @@ "@types/react-beautiful-dnd": "^13.0.0", "@types/react-dom": "^17.0.1", "@types/styled-components": "^5.1.7", + "customize-cra": "^1.0.0", "source-map-explorer": "^2.5.2", - "typescript": "^4.8.0-dev.20220630" + "typescript": "^4.7.4" }, "scripts": { "start": "react-app-rewired start", diff --git a/packages/rath-client/src/components/radarChart.tsx b/packages/rath-client/src/components/radarChart.tsx index dd0d4ba4..1838b0b8 100644 --- a/packages/rath-client/src/components/radarChart.tsx +++ b/packages/rath-client/src/components/radarChart.tsx @@ -1,7 +1,7 @@ import React, { useRef, useEffect, useMemo } from 'react'; import embed from 'vega-embed'; import { scheme } from 'vega'; -import { DefaultIWorker } from "visual-insights/build/esm/insights/dev"; +import { DefaultIWorker } from "visual-insights"; scheme('threshold', ['#1890ff', '#ffccc7']); diff --git a/packages/rath-client/src/dev/extend.ts b/packages/rath-client/src/dev/extend.ts new file mode 100644 index 00000000..3c9b8b82 --- /dev/null +++ b/packages/rath-client/src/dev/extend.ts @@ -0,0 +1,6 @@ +import { IFieldMeta, IRow } from "../interfaces"; + +export function temporalExtend (dataSource: IRow[], fields: IFieldMeta[]) { + const temporalFields = fields.filter(f => f.semanticType === 'temporal'); + +} \ No newline at end of file diff --git a/packages/rath-client/src/dev/index.ts b/packages/rath-client/src/dev/index.ts index bf55be59..1418ad12 100644 --- a/packages/rath-client/src/dev/index.ts +++ b/packages/rath-client/src/dev/index.ts @@ -1,9 +1,10 @@ -import { IRow } from "visual-insights"; -import { entropy, getCombination } from "visual-insights/build/esm/statistics"; +import { IRow, Statistics } from "visual-insights"; import { IFieldMeta } from "../interfaces"; import { getRange } from "../utils"; import { bin, binMapShareRange, binShareRange, generalMatMic, generalMic, incSim, l1Dis2, mic, normalizeScatter, rangeNormilize } from "./utils"; +const { entropy, getCombination } = Statistics; + export interface IFilter { field: IFieldMeta; values: any[]; diff --git a/packages/rath-client/src/dev/utils.ts b/packages/rath-client/src/dev/utils.ts index 0bdb618d..32a9a16e 100644 --- a/packages/rath-client/src/dev/utils.ts +++ b/packages/rath-client/src/dev/utils.ts @@ -1,6 +1,13 @@ -import { entropy } from "visual-insights/build/esm/statistics"; +// @ts-ignore +import * as vi from 'visual-insights/lib/esm/index' import { getRange } from "../utils"; +function entropy (p: number[]) { + return 0 +} + +console.log(vi, vi.Statistics) + export function firstWDis (p1: number[], p2: number[]) { } diff --git a/packages/rath-client/src/pages/dataSource/dataTable/headerCell.tsx b/packages/rath-client/src/pages/dataSource/dataTable/headerCell.tsx index 7e476386..e1835bd0 100644 --- a/packages/rath-client/src/pages/dataSource/dataTable/headerCell.tsx +++ b/packages/rath-client/src/pages/dataSource/dataTable/headerCell.tsx @@ -5,7 +5,7 @@ import intl from 'react-intl-universal'; import DistributionChart from '../metaView/distChart'; import DropdownSelect from '../../../components/dropDownSelect' import { IFieldMeta, IRawField } from '../../../interfaces'; -import { IAnalyticType, ISemanticType } from 'visual-insights/build/esm/insights/InsightFlow/interfaces'; +import { IAnalyticType, ISemanticType } from 'visual-insights'; import { Callout, IconButton, TextField } from 'office-ui-fabric-react'; import { useId } from '@uifabric/react-hooks'; diff --git a/packages/rath-client/src/pages/dev/index.tsx b/packages/rath-client/src/pages/dev/index.tsx index 4e0abd00..4deb4cfd 100644 --- a/packages/rath-client/src/pages/dev/index.tsx +++ b/packages/rath-client/src/pages/dev/index.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect, useMemo } from 'react'; import styled from 'styled-components'; import intl from 'react-intl-universal' -import { InsightSpace, DefaultIWorker } from 'visual-insights/build/esm/insights/dev'; +import { InsightSpace, DefaultIWorker } from 'visual-insights'; import { specification } from "visual-insights"; import { getInsightViewSpace } from '../../service'; import { PrimaryButton, SpinButton, Slider, ProgressIndicator } from "office-ui-fabric-react"; diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx index 43fac318..63739ee7 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx @@ -12,7 +12,7 @@ const FilterSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); const { filterSpecList, computing, filterViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; const loadMore = useCallback(() => { - discoveryMainStore.increaseRenderAmount('featViews'); + discoveryMainStore.increaseRenderAmount('filterViews'); }, [discoveryMainStore]) const recommandFilter = useCallback(() => { discoveryMainStore.filterAssociate(); diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx index ca321e8a..274b9f2b 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx @@ -12,7 +12,7 @@ const PattSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); const { pattSpecList, computing, pattViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; const loadMore = useCallback(() => { - discoveryMainStore.increaseRenderAmount('featViews'); + discoveryMainStore.increaseRenderAmount('pattViews'); }, [discoveryMainStore]) const assViews = useCallback(() => { discoveryMainStore.pattAssociate(); diff --git a/packages/rath-client/src/queries/baseVis.ts b/packages/rath-client/src/queries/baseVis.ts index a8e8d4d7..c7d70834 100644 --- a/packages/rath-client/src/queries/baseVis.ts +++ b/packages/rath-client/src/queries/baseVis.ts @@ -1,5 +1,5 @@ import { ISemanticType } from "visual-insights"; -import { Specification } from "visual-insights/build/esm/commonTypes"; +import { Specification } from "visual-insights"; import { IFieldMeta, IResizeMode, IRow } from "../interfaces"; import { applySizeConfig } from "./base/utils"; export const geomTypeMap: { [key: string]: any } = { diff --git a/packages/rath-client/src/queries/commonVis.ts b/packages/rath-client/src/queries/commonVis.ts index 413d5287..77f0d3a3 100644 --- a/packages/rath-client/src/queries/commonVis.ts +++ b/packages/rath-client/src/queries/commonVis.ts @@ -1,5 +1,5 @@ import { ISemanticType } from "visual-insights"; -import { Specification } from "visual-insights/build/esm/commonTypes"; +import { Specification } from "visual-insights"; import { IFieldMeta, IRow } from "../interfaces"; export const geomTypeMap: { [key: string]: any } = { interval: "bar", diff --git a/packages/rath-client/src/queries/featureVis.ts b/packages/rath-client/src/queries/featureVis.ts index 8214b39a..abea2dca 100644 --- a/packages/rath-client/src/queries/featureVis.ts +++ b/packages/rath-client/src/queries/featureVis.ts @@ -1,4 +1,4 @@ -import { Specification } from "visual-insights/build/esm/commonTypes"; +import { Specification } from "visual-insights"; import { Aggregator } from "../global"; import { geomTypeMap, DataField } from './index'; import { inferFieldSemanticTypeWithDict, inferFieldTypeWithDict } from "./utils"; diff --git a/packages/rath-client/src/queries/index.ts b/packages/rath-client/src/queries/index.ts index d46fc720..b5eeca12 100644 --- a/packages/rath-client/src/queries/index.ts +++ b/packages/rath-client/src/queries/index.ts @@ -1,4 +1,4 @@ -import { FieldType } from 'visual-insights/build/esm/commonTypes' +import { FieldType } from 'visual-insights' import { featureVis } from './featureVis' import { targetVis } from './targetVis'; import { baseVis } from './baseVis'; diff --git a/packages/rath-client/src/queries/labdistVis.ts b/packages/rath-client/src/queries/labdistVis.ts index b547316c..7734c4fd 100644 --- a/packages/rath-client/src/queries/labdistVis.ts +++ b/packages/rath-client/src/queries/labdistVis.ts @@ -1,7 +1,7 @@ /** * distVis 是分布式可视化的推荐,是比较新的模块,目前暂时用于dev模块,即voyager模式下的测试。 */ -import { entropy } from "visual-insights/build/esm/statistics"; +import { Statistics } from 'visual-insights' import { IPattern } from "../dev"; import { bin, binMap, mic, pureGeneralMic, rangeNormilize } from "../dev/utils"; import { IFieldMeta, IResizeMode, IRow } from "../interfaces"; @@ -436,7 +436,7 @@ export function labDistVis(props: BaseVisProps) { } score /= (measures.length - 1) } else { - score = Math.log2(16) - entropy(rangeNormilize(bin(values1).filter(v => v > 0))) + score = Math.log2(16) - Statistics.entropy(rangeNormilize(bin(values1).filter(v => v > 0))) } measures[i].features.entropy = score; } diff --git a/packages/rath-client/src/queries/targetVis.ts b/packages/rath-client/src/queries/targetVis.ts index 02d3ad9a..cfd3910e 100644 --- a/packages/rath-client/src/queries/targetVis.ts +++ b/packages/rath-client/src/queries/targetVis.ts @@ -1,4 +1,4 @@ -import { Specification } from 'visual-insights/build/esm/commonTypes' +import { Specification } from 'visual-insights' import { Aggregator } from '../global'; import { geomTypeMap, DataField } from './index'; import { inferFieldSemanticTypeWithDict, inferFieldTypeWithDict } from './utils'; diff --git a/packages/rath-client/src/queries/utils.ts b/packages/rath-client/src/queries/utils.ts index 63eeaca7..8f473317 100644 --- a/packages/rath-client/src/queries/utils.ts +++ b/packages/rath-client/src/queries/utils.ts @@ -1,4 +1,4 @@ -import { FieldType } from "visual-insights/build/esm/commonTypes"; +import { FieldType } from "visual-insights"; import { DataField } from "./index"; interface FieldTypeDictonary { [key: string]: DataField diff --git a/packages/rath-client/src/service.ts b/packages/rath-client/src/service.ts index 06616707..55b76489 100644 --- a/packages/rath-client/src/service.ts +++ b/packages/rath-client/src/service.ts @@ -43,7 +43,7 @@ import CleanWorker from './workers/clean.worker?worker'; // @ts-ignore // eslint-disable-next-line import FilterWorker from './workers/filterData.worker?worker'; -import { InsightSpace } from 'visual-insights/build/esm/insights/dev'; +import { InsightSpace } from 'visual-insights'; import { MessageProps } from './workers/engine/service'; import { CleanMethod, IFieldMeta, IFilter, IMuteFieldBase, IRawField, IRow } from './interfaces'; diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index a71b003b..021a850e 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -2,7 +2,7 @@ import { makeAutoObservable, observable, runInAction, toJS } from "mobx"; import { fromStream, IStreamListener, toStream } from "mobx-utils"; import { combineLatest, from, Subscription } from "rxjs"; import * as op from 'rxjs/operators' -import { IAnalyticType, ISemanticType } from "visual-insights/build/esm/insights/InsightFlow/interfaces"; +import { IAnalyticType, ISemanticType } from "visual-insights"; import { notify } from "../components/error"; import { RATH_INDEX_COLUMN_KEY } from "../constants"; import { IDataPreviewMode, IDatasetBase, IFieldMeta, IMuteFieldBase, IRawField, IRow, IFilter, CleanMethod, IDataPrepProgressTag } from "../interfaces"; diff --git a/packages/rath-client/src/store/exploreStore.ts b/packages/rath-client/src/store/exploreStore.ts index fe4e9ee6..94868fd7 100644 --- a/packages/rath-client/src/store/exploreStore.ts +++ b/packages/rath-client/src/store/exploreStore.ts @@ -1,6 +1,6 @@ import { computed, makeAutoObservable, observable, runInAction, toJS } from 'mobx'; import { Specification, IInsightSpace } from 'visual-insights'; -import { ISpec } from 'visual-insights/build/esm/insights/InsightFlow/specification/encoding'; +import { ISpec } from 'visual-insights'; import { STORAGE_FILE_SUFFIX } from '../constants'; import { Aggregator } from '../global'; import { IResizeMode, IRow, ITaskTestMode, PreferencePanelConfig } from '../interfaces'; @@ -9,6 +9,7 @@ import { isSetEqual } from '../utils'; import { RathStorageDump } from '../utils/storage'; import { LTSPipeLine } from './pipeLineStore/lts'; + export interface IVizSpace extends IInsightSpace { schema: Specification; dataView: IRow[] diff --git a/packages/rath-client/src/store/pipeLineStore/lts.ts b/packages/rath-client/src/store/pipeLineStore/lts.ts index 1b692c95..0dfc78bf 100644 --- a/packages/rath-client/src/store/pipeLineStore/lts.ts +++ b/packages/rath-client/src/store/pipeLineStore/lts.ts @@ -1,6 +1,5 @@ import { makeAutoObservable, observable, runInAction } from "mobx"; -import { ICubeStorageManageMode, Sampling, Specification } from "visual-insights"; -import { IFieldSummary, IInsightSpace } from "visual-insights/build/esm/insights/InsightFlow/interfaces"; +import { ICubeStorageManageMode, Sampling, Specification, IFieldSummary, IInsightSpace } from "visual-insights"; import { IRow, ISyncEngine, ITaskTestMode } from "../../interfaces"; import { IVizSpace } from "../../pages/megaAutomation/association/assCharts"; diff --git a/packages/rath-client/src/utils/transform.ts b/packages/rath-client/src/utils/transform.ts index bd29ad0b..c9b49e2e 100644 --- a/packages/rath-client/src/utils/transform.ts +++ b/packages/rath-client/src/utils/transform.ts @@ -1,7 +1,6 @@ import dayjs from "dayjs" import customParseFormat from "dayjs/plugin/customParseFormat" -import { IRow } from "visual-insights" -import { IAnalyticType, ISemanticType } from "visual-insights/build/esm/insights/InsightFlow/interfaces" +import { IRow, IAnalyticType, ISemanticType } from "visual-insights" import { inferAnalyticTypeFromSemanticType } from "." import { Field } from "../global" import { IFieldMeta, IGeoRole } from "../interfaces" diff --git a/packages/rath-client/src/visBuilder/vegaBase.tsx b/packages/rath-client/src/visBuilder/vegaBase.tsx index 7c421e0b..301e67d7 100644 --- a/packages/rath-client/src/visBuilder/vegaBase.tsx +++ b/packages/rath-client/src/visBuilder/vegaBase.tsx @@ -6,7 +6,6 @@ import { baseVis, commonVis } from '../queries/index'; import { EDITOR_URL } from '../constants'; import { IFieldMeta, IResizeMode, IRow } from '../interfaces'; -// import { simpleAggregate } from 'visual-insights/build/esm/statistics'; export const geomTypeMap: {[key: string]: any} = { interval: 'bar', line: 'line', diff --git a/packages/rath-client/src/workers/dev.worker.js b/packages/rath-client/src/workers/dev.worker.js index cacec6be..3fac0936 100644 --- a/packages/rath-client/src/workers/dev.worker.js +++ b/packages/rath-client/src/workers/dev.worker.js @@ -1,5 +1,5 @@ /* eslint no-restricted-globals: 0 */ -import { getVisSpaces } from 'visual-insights/build/esm/insights/dev'; +import { getVisSpaces } from 'visual-insights'; import { timer } from './timer'; const generateDashBoard = async (e) => { diff --git a/packages/rath-client/src/workers/engine/cleaner.ts b/packages/rath-client/src/workers/engine/cleaner.ts index 12e4ea2e..689bfe43 100644 --- a/packages/rath-client/src/workers/engine/cleaner.ts +++ b/packages/rath-client/src/workers/engine/cleaner.ts @@ -28,10 +28,10 @@ function transformDataTypes (dataSource: IRow[], fields: IRawField[]): IRow[] { }); } - export function cleanAndTransformData (dataSource: IRow[], fields: IRawField[], method: CleanMethod): IRow[] { +export function cleanAndTransformData (dataSource: IRow[], fields: IRawField[], method: CleanMethod): IRow[] { const dimensions = fields.filter(f => f.analyticType === 'dimension').map(f => f.fid); const measures = fields.filter(f => f.analyticType === 'measure').map(f => f.fid); const t = transformDataTypes(dataSource, fields) const c = cleanData(t, dimensions, measures, method) return c - } \ No newline at end of file +} \ No newline at end of file diff --git a/packages/rath-client/src/workers/engine/core.ts b/packages/rath-client/src/workers/engine/core.ts index 1de8575d..02658b27 100644 --- a/packages/rath-client/src/workers/engine/core.ts +++ b/packages/rath-client/src/workers/engine/core.ts @@ -1,14 +1,11 @@ -import { IInsightSpace, Insight } from 'visual-insights' -import { Cube } from 'visual-insights/build/esm/cube'; -import { DataGraph } from 'visual-insights/build/esm/insights/InsightFlow/dataGraph'; -import { ViewSpace } from 'visual-insights/build/esm/insights/InsightFlow/engine'; -import { KNNClusterWorker } from 'visual-insights/build/esm/insights/workers/KNNCluster'; +import { IInsightSpace, InsightFlow, Cube, ViewSpace, } from 'visual-insights' import { IRow } from '../../interfaces'; import { IVizSpace } from '../../store/exploreStore'; import { isSetEqual } from '../../utils'; import { intersect } from './utils'; -const VIEngine = Insight.VIEngine; +const { VIEngine, DataGraph } = InsightFlow + export function entropyAcc (fl: number[]) { let total = 0; for (let i = 0; i < fl.length; i++) { @@ -25,42 +22,42 @@ export function entropyAcc (fl: number[]) { class CustomDataGraph extends DataGraph { - public clusterDGraph(dataSource: IRow[], CORRELATION_THRESHOLD?: number) { - const { dimensions, DIMENSION_CORRELATION_THRESHOLD } = this; - // console.log(JSON.stringify(this.DG)) - // this.DClusters = getDimClusterGroups( - // this.DG, - // dimensions, - // CORRELATION_THRESHOLD || DIMENSION_CORRELATION_THRESHOLD - // ); - const threshold = CORRELATION_THRESHOLD || DIMENSION_CORRELATION_THRESHOLD; - const DG = this.DG; - const clusters: string[][] = []; - for (let i = 0; i < dimensions.length; i++) { - const groups: string[] = [] - for (let j = 0; j < dimensions.length; j++) { - if (DG[i][j] >= threshold) { - groups.push(dimensions[j]); - } - } - clusters.push(groups) - } - let uniqueClusters: string[][] = []; - for (let i = 0; i < clusters.length; i++) { - let unique = true - for (let j = i + 1; j < clusters.length; j++) { - if (isSetEqual(clusters[i], clusters[j])) { - unique = false; - break; - } - } - if (unique) { - uniqueClusters.push(clusters[i]) - } - } - this.DClusters = uniqueClusters - return uniqueClusters//this.DClusters; - } + // public clusterDGraph(dataSource: IRow[], CORRELATION_THRESHOLD?: number) { + // const { dimensions, DIMENSION_CORRELATION_THRESHOLD } = this; + // // console.log(JSON.stringify(this.DG)) + // // this.DClusters = getDimClusterGroups( + // // this.DG, + // // dimensions, + // // CORRELATION_THRESHOLD || DIMENSION_CORRELATION_THRESHOLD + // // ); + // const threshold = CORRELATION_THRESHOLD || DIMENSION_CORRELATION_THRESHOLD; + // const DG = this.DG; + // const clusters: string[][] = []; + // for (let i = 0; i < dimensions.length; i++) { + // const groups: string[] = [] + // for (let j = 0; j < dimensions.length; j++) { + // if (DG[i][j] >= threshold) { + // groups.push(dimensions[j]); + // } + // } + // clusters.push(groups) + // } + // let uniqueClusters: string[][] = []; + // for (let i = 0; i < clusters.length; i++) { + // let unique = true + // for (let j = i + 1; j < clusters.length; j++) { + // if (isSetEqual(clusters[i], clusters[j])) { + // unique = false; + // break; + // } + // } + // if (unique) { + // uniqueClusters.push(clusters[i]) + // } + // } + // this.DClusters = uniqueClusters + // return uniqueClusters//this.DClusters; + // } public clusterMGraph(dataSource: IRow[], CORRELATION_THRESHOLD?: number) { const { measures, MEASURE_CORRELATION_THRESHOLD } = this; // console.log(JSON.stringify(this.MG)) @@ -123,8 +120,8 @@ class CustomDataGraph extends DataGraph { export class RathEngine extends VIEngine { public constructor() { super(); - this.workerCollection.register('clusters', KNNClusterWorker); - this.workerCollection.enable('clusters', true); + // this.workerCollection.register('clusters', KNNClusterWorker); + // this.workerCollection.enable('clusters', true); // this.DIMENSION_NUM_IN_VIEW = { // MIN: 0, // MAX: 3 @@ -141,9 +138,6 @@ export class RathEngine extends VIEngine { dataSource, ops: aggregators }); - if (typeof injectCube === 'undefined') { - cube.storage.env = this.env; - } await cube.buildBaseCuboid(); await Promise.all(dataGraph.DClusters.map((group) => cube.buildCuboidOnCluster(group))) this.cube = cube; diff --git a/packages/rath-client/src/workers/engine/index.worker.js b/packages/rath-client/src/workers/engine/index.worker.js index 35b3dad8..695719ce 100644 --- a/packages/rath-client/src/workers/engine/index.worker.js +++ b/packages/rath-client/src/workers/engine/index.worker.js @@ -13,6 +13,11 @@ function main (e) { success: false, message: `[worker]${message}` }) + }).catch(err => { + self.postMessage({ + success: false, + message: `[worker]${err}\n${err.stack}` + }) }) } diff --git a/packages/rath-client/src/workers/engine/service.ts b/packages/rath-client/src/workers/engine/service.ts index 4685734d..29df4fe6 100644 --- a/packages/rath-client/src/workers/engine/service.ts +++ b/packages/rath-client/src/workers/engine/service.ts @@ -1,13 +1,12 @@ -import { ICubeStorageManageMode, IFieldSummary, IInsightSpace } from "visual-insights"; -import { Cube } from "visual-insights/build/esm/cube"; -import { ViewSpace } from "visual-insights/build/esm/insights/InsightFlow/engine"; -import { StatFuncName } from "visual-insights/build/esm/statistics"; +import { ICubeStorageManageMode, IFieldSummary, IInsightSpace, Cube, ViewSpace, StatFuncName } from "visual-insights"; + import { IFieldMeta, IRow, ISyncEngine } from "../../interfaces"; import { IRathStorage } from "../../utils/storage"; import { RathCHEngine } from "./clickhouse"; // import { isSetEqual } from "../../utils/index"; import { RathEngine } from "./core"; + const EngineRef: { current: RathEngine | null, mode: 'webworker' } | { current: RathCHEngine | null, mode: 'clickhouse' } = { mode: 'webworker', current: null @@ -81,7 +80,6 @@ async function startPipeLine (props: StartPipeLineProps) { cubeStorageManageMode: props.cubeStorageManageMode, dataSource: engine.dataSource }) - cube.storage.env = 'browser'; await engine.buildCube(cube); times.push(performance.now()) prints.push({ task: 'cube', value: times[times.length - 1] - times[times.length - 2] }) diff --git a/packages/rath-client/src/workers/fieldsSummary.worker.js b/packages/rath-client/src/workers/fieldsSummary.worker.js index b469a0e8..f843f4aa 100644 --- a/packages/rath-client/src/workers/fieldsSummary.worker.js +++ b/packages/rath-client/src/workers/fieldsSummary.worker.js @@ -1,6 +1,6 @@ /* eslint no-restricted-globals: 0 */ import { UnivariateSummary } from 'visual-insights'; -import { entropy } from 'visual-insights/build/esm/statistics'; +import { Statistics } from 'visual-insights'; import { bin, rangeNormilize } from '../dev/utils'; import { timer } from './timer'; @@ -31,7 +31,7 @@ const fieldSummary = (e) => { for (let i = 0; i < fieldEntropyList.length; i++) { if (fieldTypeList[i].type === 'quantitative') { const bins = bin(dataSource.map(r => r[fieldEntropyList[i].fieldName])); - fieldEntropyList[i].entropy = entropy(rangeNormilize(bins.filter(b => b > 0))); + fieldEntropyList[i].entropy = Statistics.entropy(rangeNormilize(bins.filter(b => b > 0))); fieldEntropyList[i].maxEntropy = Math.log2(16) } } diff --git a/packages/rath-client/tsconfig.json b/packages/rath-client/tsconfig.json index f71134ca..038fde0e 100644 --- a/packages/rath-client/tsconfig.json +++ b/packages/rath-client/tsconfig.json @@ -1,10 +1,10 @@ { "compilerOptions": { - "target": "es2015", + "target": "ES2015", "lib": [ "dom", "dom.iterable", - "esnext" + "esnext", ], "allowJs": true, "skipLibCheck": true, @@ -24,5 +24,8 @@ }, "include": [ "src" + ], + "exclude": [ + "**/node_modules/**" ] } diff --git a/yarn.lock b/yarn.lock index 050a889d..bdedd1c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5875,6 +5875,13 @@ cube-core@^2.13.0: resolved "https://registry.npm.taobao.org/cube-core/download/cube-core-2.13.0.tgz#d64abcfccf0765879f226e892cc32b934c3580c6" integrity sha1-1kq8/M8HZYefIm6JLMMrk0w1gMY= +customize-cra@^1.0.0: + version "1.0.0" + resolved "https://registry.npmmirror.com/customize-cra/-/customize-cra-1.0.0.tgz#73286563631aa08127ad4d30a2e3c89cf4e93c8d" + integrity sha512-DbtaLuy59224U+xCiukkxSq8clq++MOtJ1Et7LED1fLszWe88EoblEYFBJ895sB1mC6B4uu3xPT/IjClELhMbA== + dependencies: + lodash.flow "^3.5.0" + cyclist@^1.0.1: version "1.0.1" resolved "https://registry.nlark.com/cyclist/download/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" @@ -10269,7 +10276,7 @@ lodash.debounce@^4.0.8: resolved "https://registry.nlark.com/lodash.debounce/download/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= -lodash.flow@^3.3.0: +lodash.flow@^3.3.0, lodash.flow@^3.5.0: version "3.5.0" resolved "https://registry.npm.taobao.org/lodash.flow/download/lodash.flow-3.5.0.tgz#87bf40292b8cf83e4e8ce1a3ae4209e20071675a" integrity sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o= @@ -15129,10 +15136,10 @@ typescript@^4.5.4: resolved "https://registry.npmmirror.com/typescript/-/typescript-4.6.3.tgz#eefeafa6afdd31d725584c67a0eaba80f6fc6c6c" integrity sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw== -typescript@^4.8.0-dev.20220630: - version "4.8.0-dev.20220630" - resolved "https://registry.npmmirror.com/typescript/-/typescript-4.8.0-dev.20220630.tgz#7ed7c5c95d76f8236271d864fbbc93115ae0d436" - integrity sha512-wyFZIoUdTqDrKU3XfDx+pylAEIhlmpbsbicFXuvggpKeSWiPeUrMiSRqeXBnAjXiD6kn2Anmw+c3eyX3A2WdUw== +typescript@^4.7.4: + version "4.7.4" + resolved "https://registry.npmmirror.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" + integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== ua-parser-js@^0.7.18: version "0.7.28" @@ -15949,17 +15956,14 @@ visual-insights@0.7.15: cube-core "^2.13.0" simple-statistics "^7.1.0" -visual-insights@0.8.11: - version "0.8.11" - resolved "https://registry.npmmirror.com/visual-insights/-/visual-insights-0.8.11.tgz#204408b7be25d52bd68e3ff13d15e7b234957416" - integrity sha512-yMGkBJzcEIeiWKmD1uiJ4TJMXXBRMz+iIgyTBtDvDIMrAlakRF90gZzjuRidjq95FX+wfliDHkbkHkomyLdCgA== +visual-insights@0.9.11: + version "0.9.11" + resolved "https://registry.npmmirror.com/visual-insights/-/visual-insights-0.9.11.tgz#6b7548bb2f30105c85c60cb7a616d28e6b1bb9a2" + integrity sha512-5FUIAca8WavRMSLWjVdlN+OL7zGOdHjVPQQKmdf6a6bYd0ttLqXxehR4L8hXeLlz0zq3O6ZMJwEN7+b+OaNTAw== dependencies: - assert "^2.0.0" axios "^0.22.0" - cube-core "^2.13.0" detect-browser "^5.3.0" localforage "^1.10.0" - simple-statistics "^7.1.0" vite@^2.4.4: version "2.5.0" From 5034cdb81c9f65dfa88f5d99e936be302fda7c0c Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Tue, 26 Jul 2022 22:57:46 +0800 Subject: [PATCH 06/15] =?UTF-8?q?feat:=20=E4=B8=80=E4=B8=AA=E5=8F=AF?= =?UTF-8?q?=E7=94=A8=E7=9A=84=E5=8F=91=E5=B8=83=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 3 +- packages/rath-client/config-overrides.js | 10 ++-- packages/rath-client/package.json | 2 +- packages/rath-client/src/dev/utils.ts | 4 -- .../rath-client/src/store/dataSourceStore.ts | 12 +++-- packages/rath-client/tsconfig.json | 6 +-- yarn.lock | 50 ++++++++++--------- 7 files changed, 47 insertions(+), 40 deletions(-) diff --git a/package.json b/package.json index 24eeb947..a746babd 100644 --- a/package.json +++ b/package.json @@ -40,5 +40,6 @@ "devDependencies": { "concurrently": "^4.1.2", "yarn": "^1.19.0" - } + }, + "dependencies": {} } diff --git a/packages/rath-client/config-overrides.js b/packages/rath-client/config-overrides.js index bec69d3e..674cc350 100644 --- a/packages/rath-client/config-overrides.js +++ b/packages/rath-client/config-overrides.js @@ -1,5 +1,5 @@ -const path = require('path'); -const { override, babelInclude } = require('customize-cra') +// const path = require('path'); +// const { override, babelInclude } = require('customize-cra') // const mid_override = override( // babelInclude([ @@ -8,7 +8,7 @@ const { override, babelInclude } = require('customize-cra') // ]) // ) -module.exports = function final_override(config, env) { +module.exports = function override(config, env) { // do stuff with the webpack config... config.module.rules.push({ test: /\.worker\.js$/, @@ -21,8 +21,8 @@ module.exports = function final_override(config, env) { }, }) config.output.globalObject = 'self' - config.module = config.module || {}; - config.module.unknownContextCritical = false + // config.module = config.module || {}; + // config.module.unknownContextCritical = false return config // return mid_override(config) }; \ No newline at end of file diff --git a/packages/rath-client/package.json b/packages/rath-client/package.json index a3336d3c..2f0c12d3 100644 --- a/packages/rath-client/package.json +++ b/packages/rath-client/package.json @@ -45,7 +45,7 @@ "vega": "^5.22.1", "vega-embed": "^6.20.8", "vega-lite": "^5.2.0", - "visual-insights": "0.9.11", + "visual-insights": "0.11.2", "web-vitals": "^0.2.4", "worker-loader": "^3.0.7" }, diff --git a/packages/rath-client/src/dev/utils.ts b/packages/rath-client/src/dev/utils.ts index 32a9a16e..ea3f82f6 100644 --- a/packages/rath-client/src/dev/utils.ts +++ b/packages/rath-client/src/dev/utils.ts @@ -1,13 +1,9 @@ -// @ts-ignore -import * as vi from 'visual-insights/lib/esm/index' import { getRange } from "../utils"; function entropy (p: number[]) { return 0 } -console.log(vi, vi.Statistics) - export function firstWDis (p1: number[], p2: number[]) { } diff --git a/packages/rath-client/src/store/dataSourceStore.ts b/packages/rath-client/src/store/dataSourceStore.ts index 021a850e..672a929f 100644 --- a/packages/rath-client/src/store/dataSourceStore.ts +++ b/packages/rath-client/src/store/dataSourceStore.ts @@ -144,13 +144,19 @@ export class DataSourceStore { } }) this.subscriptions.push(rawData$.subscribe(() => { - this.dataPrepProgressTag = IDataPrepProgressTag.filter; + runInAction(() => { + this.dataPrepProgressTag = IDataPrepProgressTag.filter; + }) })) this.subscriptions.push(filteredData$.subscribe(() => { - this.dataPrepProgressTag = IDataPrepProgressTag.clean + runInAction(() => { + this.dataPrepProgressTag = IDataPrepProgressTag.clean + }) })) this.subscriptions.push(cleanedData$.subscribe(() => { - this.dataPrepProgressTag = IDataPrepProgressTag.none; + runInAction(() => { + this.dataPrepProgressTag = IDataPrepProgressTag.none; + }) })) } diff --git a/packages/rath-client/tsconfig.json b/packages/rath-client/tsconfig.json index 038fde0e..b99f09ae 100644 --- a/packages/rath-client/tsconfig.json +++ b/packages/rath-client/tsconfig.json @@ -25,7 +25,7 @@ "include": [ "src" ], - "exclude": [ - "**/node_modules/**" - ] + // "exclude": [ + // "**/node_modules/**" + // ] } diff --git a/yarn.lock b/yarn.lock index bdedd1c1..37a057c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2237,6 +2237,13 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" +"@kanaries/adapters@1.0.3": + version "1.0.3" + resolved "https://registry.npmmirror.com/@kanaries/adapters/-/adapters-1.0.3.tgz#67704fd94eca59d05edea5f098c8cb867b026dc5" + integrity sha512-OlrMGI/w6U5TE9hBH0hH0/fqG/OS0y0OyE1Nvq/vqZXskfLyxmxrRQsSnUvhOe2JgsGi/EtSB4iSh7+rZaF0TA== + dependencies: + localforage "^1.10.0" + "@kanaries/web-data-loader@0.1.5": version "0.1.5" resolved "https://registry.npmmirror.com/@kanaries/web-data-loader/-/web-data-loader-0.1.5.tgz#10d8d677492919b0838b9429d28cbf2b7d726100" @@ -4896,20 +4903,10 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001219: - version "1.0.30001241" - resolved "https://registry.nlark.com/caniuse-lite/download/caniuse-lite-1.0.30001241.tgz#cd3fae47eb3d7691692b406568d7a3e5b23c7598" - integrity sha1-zT+uR+s9dpFpK0BlaNej5bI8dZg= - -caniuse-lite@^1.0.30001259: - version "1.0.30001259" - resolved "https://registry.nlark.com/caniuse-lite/download/caniuse-lite-1.0.30001259.tgz?cache=0&sync_timestamp=1632203439345&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fcaniuse-lite%2Fdownload%2Fcaniuse-lite-1.0.30001259.tgz#ae21691d3da9c4be6144403ac40f71d9f6efd790" - integrity sha1-riFpHT2pxL5hREA6xA9x2fbv15A= - -caniuse-lite@^1.0.30001317: - version "1.0.30001322" - resolved "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001322.tgz#2e4c09d11e1e8f852767dab287069a8d0c29d623" - integrity sha512-neRmrmIrCGuMnxGSoh+x7zYtQFFgnSY2jaomjU56sCkTA6JINqQrxutF459JpWcWRajvoyn95sOXq4Pqrnyjew== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001259, caniuse-lite@^1.0.30001317: + version "1.0.30001370" + resolved "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001370.tgz" + integrity sha512-3PDmaP56wz/qz7G508xzjx8C+MC2qEm4SYhSEzC9IBROo+dGXFWRuaXkWti0A9tuI00g+toiriVqxtWMgl350g== capture-exit@^2.0.0: version "2.0.0" @@ -15013,6 +15010,11 @@ tslib@^2.1.0, tslib@^2.3.1, tslib@~2.3.1: resolved "https://registry.npmmirror.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== +tslib@^2.4.0: + version "2.4.0" + resolved "https://registry.npmmirror.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" + integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== + tslib@~2.1.0: version "2.1.0" resolved "https://registry.nlark.com/tslib/download/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" @@ -15946,6 +15948,17 @@ vfile@^4.0.0: unist-util-stringify-position "^2.0.0" vfile-message "^2.0.0" +visual-insights@0.11.2: + version "0.11.2" + resolved "https://registry.npmmirror.com/visual-insights/-/visual-insights-0.11.2.tgz#c10f29aeeeca3e35c116b8af2c3ae8e4e4531088" + integrity sha512-MFQfIPLDVvCTQ+Mw7aUw5ZOl8sRe5kK4yPsGDH3bsVFwG4KIwYXpTu0tNehwkZC5JG27wAQ7yXM2nBT6HbjzIA== + dependencies: + "@kanaries/adapters" "1.0.3" + axios "^0.22.0" + detect-browser "^5.3.0" + localforage "^1.10.0" + tslib "^2.4.0" + visual-insights@0.7.15: version "0.7.15" resolved "https://registry.npmmirror.com/visual-insights/-/visual-insights-0.7.15.tgz#04806c8c78739f7bb8a7ee60bb56ee8ddc7668a7" @@ -15956,15 +15969,6 @@ visual-insights@0.7.15: cube-core "^2.13.0" simple-statistics "^7.1.0" -visual-insights@0.9.11: - version "0.9.11" - resolved "https://registry.npmmirror.com/visual-insights/-/visual-insights-0.9.11.tgz#6b7548bb2f30105c85c60cb7a616d28e6b1bb9a2" - integrity sha512-5FUIAca8WavRMSLWjVdlN+OL7zGOdHjVPQQKmdf6a6bYd0ttLqXxehR4L8hXeLlz0zq3O6ZMJwEN7+b+OaNTAw== - dependencies: - axios "^0.22.0" - detect-browser "^5.3.0" - localforage "^1.10.0" - vite@^2.4.4: version "2.5.0" resolved "https://registry.nlark.com/vite/download/vite-2.5.0.tgz?cache=0&sync_timestamp=1629090689000&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fvite%2Fdownload%2Fvite-2.5.0.tgz#111ba3679432d426e44566acf480005a7914cbd6" From 8106953090064f2e9d35f1287f5de7b5f6f0a74f Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Thu, 28 Jul 2022 14:09:15 +0800 Subject: [PATCH 07/15] fix: field type infer --- .../graphic-walker/src/dataSource/config.ts | 18 +++++++++--------- packages/rath-client/src/utils/index.ts | 17 +++++++++++++++-- .../src/workers/engine/metaInfer.ts | 13 +++++++++++-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/packages/graphic-walker/src/dataSource/config.ts b/packages/graphic-walker/src/dataSource/config.ts index 05816616..d78381d4 100644 --- a/packages/graphic-walker/src/dataSource/config.ts +++ b/packages/graphic-walker/src/dataSource/config.ts @@ -9,15 +9,15 @@ export const DemoDataAssets = process.env.NODE_ENV === 'production' ? { KELPER: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-kelper-service.json', } : { // CARS: "https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-cars-service.json", - CARS: "http://localhost:8080/api/ds-cars-service.json", - // STUDENTS: "https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-students-service.json", - STUDENTS: "http://localhost:8080/api/ds-students-service.json", - BTC_GOLD: "http://localhost:8080/api/ds_btc_gold_service.json", - BIKE_SHARING: 'http://localhost:8080/api/ds-bikesharing-service.json', - CAR_SALES: 'http://localhost:8080/api/ds-carsales-service.json', - COLLAGE: 'http://localhost:8080/api/ds-collage-service.json', - TITANIC: 'http://localhost:8080/api/ds-titanic-service.json', - KELPER: 'http://localhost:8080/api/ds-kelper-service.json', + CARS: "http://localhost:8080/datasets/ds-cars-service.json", + // STUDENTS: "https://chspace.oss-cn-hongkong.aliyuncs.com/datasets/ds-students-service.json", + STUDENTS: "http://localhost:8080/datasets/ds-students-service.json", + BTC_GOLD: "http://localhost:8080/datasets/ds_btc_gold_service.json", + BIKE_SHARING: 'http://localhost:8080/datasets/ds-bikesharing-service.json', + CAR_SALES: 'http://localhost:8080/datasets/ds-carsales-service.json', + COLLAGE: 'http://localhost:8080/datasets/ds-collage-service.json', + TITANIC: 'http://localhost:8080/datasets/ds-titanic-service.json', + KELPER: 'http://localhost:8080/datasets/ds-kelper-service.json', } as const; interface IPublicData { diff --git a/packages/rath-client/src/utils/index.ts b/packages/rath-client/src/utils/index.ts index c61c1936..7feaef25 100644 --- a/packages/rath-client/src/utils/index.ts +++ b/packages/rath-client/src/utils/index.ts @@ -38,8 +38,21 @@ function inferAnalyticTypeFromSemanticType (semanticType: ISemanticType): IAnaly * @param fid 字段id * @returns semantic type 列表 */ - export function inferSemanticType (data: IRow[], fid: string) { - return UnivariateSummary.getFieldType(data, fid); + export function inferSemanticType (data: IRow[], fid: string): ISemanticType { + let st = UnivariateSummary.getFieldType(data, fid); + if (st === 'ordinal') { + const valueSet: Set = new Set(); + let _max = -Infinity; + let _min = Infinity; + for (let v of valueSet) { + _max = Math.max(_max, v) + _min = Math.max(_min, v) + } + if (_max - _min + 1 !== valueSet.size) { + st = 'quantitative' + } + } + return st; } function isSetEqual (A: string[], B: string[]) { diff --git a/packages/rath-client/src/workers/engine/metaInfer.ts b/packages/rath-client/src/workers/engine/metaInfer.ts index b033c229..e9a93137 100644 --- a/packages/rath-client/src/workers/engine/metaInfer.ts +++ b/packages/rath-client/src/workers/engine/metaInfer.ts @@ -1,6 +1,6 @@ import { IAnalyticType, IRow, ISemanticType } from "visual-insights"; import { IGeoRole, IMuteFieldBase, IRawField } from "../../interfaces"; -import { inferAnalyticType, inferSemanticType } from "../../utils"; +import { inferAnalyticType, inferAnalyticTypeFromSemanticType, inferSemanticType } from "../../utils"; export function emptyCount (dataSource: IRow[], colKey: string): number { // const counter: Map = new Map(); @@ -27,6 +27,15 @@ function inferDisable (dataSource: IRow[], colKey: string) { } } if (valueSet.size === 1) return true; + if (valueSet.size > 50 && valueSet.size === dataSource.length) { + for (let v of valueSet) { + if (typeof v !== 'string' && !Number.isInteger(v)) { + return false; + } + } + return true; + } + return false; } @@ -46,7 +55,7 @@ export function inferMeta (props: { dataSource: IRow[]; fields: IMuteFieldBase[] let geoRole = field.geoRole === '?' ? inferGeoRole(dataSource, field.fid, semanticType, field.name || '') : field.geoRole; let analyticType: IAnalyticType = 'dimension'; if (geoRole === 'none') { - analyticType = field.analyticType === '?' ? inferAnalyticType(dataSource, field.fid) : field.analyticType; + analyticType = field.analyticType === '?' ? inferAnalyticTypeFromSemanticType(semanticType) : field.analyticType; } const disable: boolean = field.disable === '?' ? inferDisable(dataSource, field.fid) : Boolean(field.disable); // TODO: 临时处理逻辑。后续可视化部分扩展好了,这部分要消除掉。(使用dist图表就可以解决) From 8ab173114a5b58d551b5fbef0a3a1759acc4d767 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Thu, 28 Jul 2022 14:56:43 +0800 Subject: [PATCH 08/15] fix: entropy is zero bug --- packages/rath-client/src/dev/index.ts | 2 ++ packages/rath-client/src/dev/utils.ts | 5 ++--- packages/rath-client/src/store/discovery/mainStore.ts | 2 -- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/rath-client/src/dev/index.ts b/packages/rath-client/src/dev/index.ts index 1418ad12..9581f11a 100644 --- a/packages/rath-client/src/dev/index.ts +++ b/packages/rath-client/src/dev/index.ts @@ -45,6 +45,8 @@ export class NextVICore { let viewDimNotInFilters: IFieldMeta[] = [...viewDimensions]; if (typeof view.filters !== 'undefined') { viewDimNotInFilters = viewDimensions.filter(vd => !view.filters!.find(vf => vf.field.fid === vd.fid)); + } else { + viewDimNotInFilters = [...viewDimensions] } const ans: IPattern[] = []; if (viewDimNotInFilters.length > 0) { diff --git a/packages/rath-client/src/dev/utils.ts b/packages/rath-client/src/dev/utils.ts index ea3f82f6..bb0d4a5d 100644 --- a/packages/rath-client/src/dev/utils.ts +++ b/packages/rath-client/src/dev/utils.ts @@ -1,8 +1,7 @@ +import { Statistics } from 'visual-insights' import { getRange } from "../utils"; -function entropy (p: number[]) { - return 0 -} +const { entropy } = Statistics; export function firstWDis (p1: number[], p2: number[]) { diff --git a/packages/rath-client/src/store/discovery/mainStore.ts b/packages/rath-client/src/store/discovery/mainStore.ts index 9c8111e5..174c7b90 100644 --- a/packages/rath-client/src/store/discovery/mainStore.ts +++ b/packages/rath-client/src/store/discovery/mainStore.ts @@ -179,7 +179,6 @@ export class DiscoveryMainStore { public async initAssociate () { this.computing = false; const { dataSource, fieldMetas } = this; - console.log('init') try { const res = await footmanEngineService({ dataSource, @@ -229,7 +228,6 @@ export class DiscoveryMainStore { this.mainView = produce(this.mainView, draft => { draft.fields.splice(targetFieldIndex, 1) }) - console.log(this.mainView) } public clearViews () { this.featViews = makeInitAssoViews(); From b5393886ee9c666906f469718a81b2aac6d22c97 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Thu, 28 Jul 2022 15:39:37 +0800 Subject: [PATCH 09/15] fix: add sort for semi-auto segment --- packages/rath-client/src/queries/labdistVis.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/rath-client/src/queries/labdistVis.ts b/packages/rath-client/src/queries/labdistVis.ts index 7734c4fd..8981428f 100644 --- a/packages/rath-client/src/queries/labdistVis.ts +++ b/packages/rath-client/src/queries/labdistVis.ts @@ -478,6 +478,19 @@ export function labDistVis(props: BaseVisProps) { fields: distFields, usedChannels, statFields, statEncodes }) + if (markType === 'bar' && statEncodes.length > 0) { + if (enc && enc.x && enc.y) { + if (enc.x.field && enc.y.field) { + const sortEncodeField = (enc.y.type === 'quantitative' ? enc.x : enc.y); + const sortBasedEncodeField = (enc.y.type === 'quantitative' ? enc.y : enc.x); + sortEncodeField.sort = { + field: sortBasedEncodeField.field, + op: sortBasedEncodeField.aggregate || 'count', + order: 'descending' + } + } + } + } // if (filters && filters.length > 0) { // const field = filters[0].field; // enc.color = { From e6a1830899861e3fd4fd384c54de0306bfe132b5 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Thu, 28 Jul 2022 17:40:18 +0800 Subject: [PATCH 10/15] feat: better nlg interaction --- .../src/components/vizPreference.tsx | 16 ++++++++- .../src/pages/dataSource/config.ts | 18 +++++----- .../pages/megaAutomation/narrative/index.tsx | 36 ++++++++++++++----- .../vizOperation/operationBar.tsx | 2 +- .../rath-client/src/queries/labdistVis.ts | 26 -------------- packages/rath-client/src/service.ts | 7 ++++ .../rath-client/src/store/exploreStore.ts | 4 +++ 7 files changed, 64 insertions(+), 45 deletions(-) diff --git a/packages/rath-client/src/components/vizPreference.tsx b/packages/rath-client/src/components/vizPreference.tsx index b55abd41..a8ffdf21 100644 --- a/packages/rath-client/src/components/vizPreference.tsx +++ b/packages/rath-client/src/components/vizPreference.tsx @@ -1,6 +1,6 @@ import React, { useCallback } from "react"; import { observer } from 'mobx-react-lite'; -import { PrimaryButton, Stack, Checkbox, Panel, PanelType, ComboBox, Label } from "office-ui-fabric-react"; +import { PrimaryButton, Stack, Checkbox, Panel, PanelType, ComboBox, Label, Slider } from "office-ui-fabric-react"; import { Aggregator } from "../global"; import { useGlobalStore } from "../store"; const checkboxStyles = () => { @@ -85,6 +85,20 @@ const PreferencePanel: React.FC = () => { }} /> + `${Math.round(value * 100)}%`} + showValue={true} + onChange={(value: number) => { + exploreStore.setNlgThreshold(value); + + }} + /> ); }; diff --git a/packages/rath-client/src/pages/dataSource/config.ts b/packages/rath-client/src/pages/dataSource/config.ts index c15b676d..bef2bbc3 100644 --- a/packages/rath-client/src/pages/dataSource/config.ts +++ b/packages/rath-client/src/pages/dataSource/config.ts @@ -61,16 +61,16 @@ export const DemoDataAssets = process.env.NODE_ENV === 'production' ? { KELPER: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-kelper-service.json', } : { // CARS: "https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-cars-service.json", - CARS: "http://localhost:3000/datasets/ds-cars-service.json", - // CARS: "http://localhost:3000/datasets/dataset-service-edge.json", + CARS: "/datasets/ds-cars-service.json", + // CARS: "/datasets/dataset-service-edge.json", // STUDENTS: "https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-students-service.json", - STUDENTS: "http://localhost:3000/datasets/ds-students-service.json", - BTC_GOLD: "http://localhost:3000/datasets/ds_btc_gold_service.json", - BIKE_SHARING: 'http://localhost:3000/datasets/ds-bikesharing-service.json', - CAR_SALES: 'http://localhost:3000/datasets/ds-carsales-service.json', - COLLAGE: 'http://localhost:3000/datasets/ds-collage-service.json', - TITANIC: 'http://localhost:3000/datasets/ds-titanic-service.json', - KELPER: 'http://localhost:3000/datasets/ds-kelper-service.json', + STUDENTS: "/datasets/ds-students-service.json", + BTC_GOLD: "/datasets/ds_btc_gold_service.json", + BIKE_SHARING: '/datasets/ds-bikesharing-service.json', + CAR_SALES: '/datasets/ds-carsales-service.json', + COLLAGE: '/datasets/ds-collage-service.json', + TITANIC: '/datasets/ds-titanic-service.json', + KELPER: '/datasets/ds-kelper-service.json', } as const; export type IDemoDataKey = keyof typeof DemoDataAssets; diff --git a/packages/rath-client/src/pages/megaAutomation/narrative/index.tsx b/packages/rath-client/src/pages/megaAutomation/narrative/index.tsx index f98a180d..136965d5 100644 --- a/packages/rath-client/src/pages/megaAutomation/narrative/index.tsx +++ b/packages/rath-client/src/pages/megaAutomation/narrative/index.tsx @@ -1,8 +1,10 @@ import { toJS } from 'mobx'; import { observer } from 'mobx-react-lite'; -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; import { useGlobalStore } from '../../../store'; import styled from 'styled-components'; +import { Spinner } from 'office-ui-fabric-react'; +import { getTestServerAPI } from '../../../service'; const InsightDesc = styled.div` margin: 4px 12px 0px 12px; @@ -36,15 +38,24 @@ const InsightDesc = styled.div` ` const Narrative: React.FC = props => { - const { exploreStore } = useGlobalStore(); - const { pageIndex, insightSpaces, dataSource, fieldMetas } = exploreStore; + const { exploreStore, langStore } = useGlobalStore(); + const { pageIndex, insightSpaces, dataSource, fieldMetas, nlgThreshold } = exploreStore; + const [explainLoading, setExplainLoading] = useState(false); + const requestId = useRef(0); const fms = toJS(fieldMetas); const fieldsInViz = useMemo(() => { return insightSpaces[pageIndex] ? [...insightSpaces[pageIndex].dimensions, ...insightSpaces[pageIndex].measures].map(fid => fms.find(fm => fm.fid === fid)) : [] }, [insightSpaces, pageIndex, fms]); const [viewInfo, setViewInfo] = useState([]) useEffect(() => { - fetch('http://localhost:8000/insight', { + setViewInfo([]) + setExplainLoading(false); + }, [pageIndex]) + useEffect(() => { + setExplainLoading(true) + requestId.current++; + let rid = requestId.current; + fetch(getTestServerAPI('insight'), { method: 'POST', headers: { 'Content-Type': 'application/json' @@ -52,21 +63,25 @@ const Narrative: React.FC = props => { body: JSON.stringify({ dataSource, fields: fieldsInViz, - aggrType: 'sum' + aggrType: 'sum', + langType: langStore.lang }) }) .then(res => res.json()) .then(res => { if (res.success) { - setViewInfo(res.data) + console.log(rid, requestId.current) + rid === requestId.current && setViewInfo(res.data) } else { throw new Error(res.message) } }).catch(err => { console.error(err); setViewInfo([]) + }).finally(() => { + setExplainLoading(false) }) - }, [pageIndex, dataSource, fieldsInViz]) + }, [pageIndex, dataSource, fieldsInViz, langStore.lang]) const explains = useMemo(() => { if (!viewInfo || viewInfo.length === 0) return [] return Object.keys(viewInfo[0]).filter((k: string) => viewInfo[0][k].score > 0).map((k: string) => ({ @@ -77,7 +92,7 @@ const Narrative: React.FC = props => { }, [viewInfo]) return
{ - explains.map(ex => + !explainLoading && explains.filter(ex => ex.score > nlgThreshold).map(ex =>
{ex.type}
{(ex.score * 100).toFixed(1)} %
@@ -86,6 +101,11 @@ const Narrative: React.FC = props => {

{ex.explain}

) } + { + explainLoading &&
+ +
+ } {/* */} {/* {JSON.stringify(viewInfo)} */}
diff --git a/packages/rath-client/src/pages/megaAutomation/vizOperation/operationBar.tsx b/packages/rath-client/src/pages/megaAutomation/vizOperation/operationBar.tsx index 529624fe..c883ff9f 100644 --- a/packages/rath-client/src/pages/megaAutomation/vizOperation/operationBar.tsx +++ b/packages/rath-client/src/pages/megaAutomation/vizOperation/operationBar.tsx @@ -117,7 +117,7 @@ const OperationBar: React.FC = props => { }} /> - { exploreStore.setVisualConig((cnf => { diff --git a/packages/rath-client/src/queries/labdistVis.ts b/packages/rath-client/src/queries/labdistVis.ts index 8981428f..c54acc9c 100644 --- a/packages/rath-client/src/queries/labdistVis.ts +++ b/packages/rath-client/src/queries/labdistVis.ts @@ -311,32 +311,6 @@ function markFixEncoding(markType: string, usedChannels: Set) { } } -// function autoAgg (props: {encoding: any; fields: IFieldMeta[]; markType: string; op?: string; statFields?: IFieldMeta[]}) { -// const { -// encoding, -// fields, -// markType, -// op = 'mean', -// statFields = [] -// } = props -// if (fields.length > 1) { -// if (markType === 'bar' || markType === 'line') { -// if (encoding.x && encoding.x.type === 'quantitative') { -// encoding.x.aggregate = op; -// if (encoding.x.title) { -// encoding.x.title = `${op}(${encoding.x.title})` -// } -// } -// if (encoding.y && encoding.y.type === 'quantitative') { -// encoding.y.aggregate = op; -// if (encoding.y.title) { -// encoding.y.title = `${op}[${encoding.y.title}]` -// } -// } -// } -// } -// } - interface IFieldEncode { field?: string; title?: string; diff --git a/packages/rath-client/src/service.ts b/packages/rath-client/src/service.ts index 55b76489..ca3aa80d 100644 --- a/packages/rath-client/src/service.ts +++ b/packages/rath-client/src/service.ts @@ -480,6 +480,13 @@ function getTestServerUrl (): URL | null { } return null; } + +export function getTestServerAPI (api: string): string { + const url = new URL(window.location.href).searchParams.get('server') || 'http://localhost:8000'; + let surl = new URL(url); + surl.pathname = api; + return surl.href; +} export async function rathEngineServerService (props: MessageServerProps) { try { const testServer = getTestServerUrl(); diff --git a/packages/rath-client/src/store/exploreStore.ts b/packages/rath-client/src/store/exploreStore.ts index 94868fd7..7ba5fb3c 100644 --- a/packages/rath-client/src/store/exploreStore.ts +++ b/packages/rath-client/src/store/exploreStore.ts @@ -44,6 +44,7 @@ export class ExploreStore { public forkView: IExploreView | null = null; public view: IExploreView | null = null; public orderBy: string = EXPLORE_VIEW_ORDER.DEFAULT; + public nlgThreshold: number = 0.2; public forkViewSpec: { schema: Specification; dataView: IRow[]; @@ -126,6 +127,9 @@ export class ExploreStore { public get samplingDataSource () { return this.ltsPipeLineStore.samplingDataSource; } + public setNlgThreshold (num: number) { + this.nlgThreshold = num; + } public setVisualConig (updater: (config: PreferencePanelConfig) => void) { runInAction(() => { updater(this.visualConfig) From 9a2afca2fd1da5ddcd308f64537523d3fcd4cac4 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Fri, 29 Jul 2022 11:57:02 +0800 Subject: [PATCH 11/15] feat: fix computing status layer --- .../predictZone/featSegment.tsx | 6 +- .../predictZone/filterSegment.tsx | 6 +- .../semiAutomation/predictZone/index.tsx | 17 +---- .../predictZone/pattSegment.tsx | 6 +- .../src/store/discovery/localTypes.ts | 4 +- .../src/store/discovery/mainStore.ts | 62 +++++++++++++------ .../rath-client/src/workers/engine/core.ts | 2 + 7 files changed, 60 insertions(+), 43 deletions(-) diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx index 3a4ad794..193e78bd 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/featSegment.tsx @@ -10,7 +10,7 @@ import { applyFilter } from '../utils'; const FeatSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); - const { featSpecList, computing, featViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; + const { featSpecList, featViews, mainVizSetting, dataSource, autoAsso, hasMainView } = discoveryMainStore; const loadMore = useCallback(() => { discoveryMainStore.increaseRenderAmount('featViews'); }, [discoveryMainStore]) @@ -24,7 +24,7 @@ const FeatSegment: React.FC = () => { !autoAsso.featViews && } @@ -32,7 +32,7 @@ const FeatSegment: React.FC = () => { { featSpecList.map((spec, i) =>
{ - computing && + featViews.computing && } diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx index 63739ee7..5aa439da 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/filterSegment.tsx @@ -10,7 +10,7 @@ import { applyFilter } from '../utils'; const FilterSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); - const { filterSpecList, computing, filterViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; + const { filterSpecList, filterViews, mainVizSetting, dataSource, autoAsso, hasMainView } = discoveryMainStore; const loadMore = useCallback(() => { discoveryMainStore.increaseRenderAmount('filterViews'); }, [discoveryMainStore]) @@ -23,7 +23,7 @@ const FilterSegment: React.FC = () => { { !autoAsso.filterViews && } @@ -31,7 +31,7 @@ const FilterSegment: React.FC = () => { { filterSpecList.map((spec, i) =>
{ - computing && + filterViews.computing && } diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx index 3309af71..51393f9a 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/index.tsx @@ -1,23 +1,10 @@ -import React, { useEffect } from 'react'; -import { observer } from 'mobx-react-lite'; - -import { useGlobalStore } from '../../../store'; +import React from 'react'; import PattSegment from './pattSegment'; import FeatSegment from './featSegment'; import FilterSegment from './filterSegment'; const PredictZone: React.FC = props => { - const { discoveryMainStore } = useGlobalStore(); - const { dataSource, fieldMetas, mainView, autoAsso } = discoveryMainStore; - const { pattViews: autoPatt, featViews: autoFeat, filterViews: autoFilter } = autoAsso; - useEffect(() => { - if (mainView) { - autoPatt && discoveryMainStore.pattAssociate(); - autoFeat && discoveryMainStore.featAssociate(); - autoFilter && discoveryMainStore.filterAssociate(); - } - }, [dataSource, fieldMetas, mainView, autoFeat, autoPatt, autoFilter, discoveryMainStore]) return
@@ -25,4 +12,4 @@ const PredictZone: React.FC = props => {
} -export default observer(PredictZone); +export default PredictZone; diff --git a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx index 274b9f2b..a30aa2eb 100644 --- a/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx +++ b/packages/rath-client/src/pages/semiAutomation/predictZone/pattSegment.tsx @@ -10,7 +10,7 @@ import { applyFilter } from '../utils'; const PattSegment: React.FC = () => { const { discoveryMainStore } = useGlobalStore(); - const { pattSpecList, computing, pattViews, mainVizSetting, dataSource, autoAsso, mainView } = discoveryMainStore; + const { pattSpecList, pattViews, mainVizSetting, dataSource, autoAsso, hasMainView } = discoveryMainStore; const loadMore = useCallback(() => { discoveryMainStore.increaseRenderAmount('pattViews'); }, [discoveryMainStore]) @@ -22,7 +22,7 @@ const PattSegment: React.FC = () => {

{intl.get('discovery.main.associate.patterns')}

{ !autoAsso.pattViews && @@ -31,7 +31,7 @@ const PattSegment: React.FC = () => { { pattSpecList.map((spec, i) =>
{ - computing && + pattViews.computing && } diff --git a/packages/rath-client/src/store/discovery/localTypes.ts b/packages/rath-client/src/store/discovery/localTypes.ts index e75d76b4..3ea51a3e 100644 --- a/packages/rath-client/src/store/discovery/localTypes.ts +++ b/packages/rath-client/src/store/discovery/localTypes.ts @@ -20,6 +20,7 @@ export type IRenderViewKey = 'pattViews' | 'featViews' | 'filterViews'; export interface IAssoViews { views: IPattern[]; amount: number; + computing: boolean; } export const RENDER_BATCH_SIZE = 5; @@ -27,6 +28,7 @@ export const RENDER_BATCH_SIZE = 5; export function makeInitAssoViews(initRenderAmount: number = 5): IAssoViews { return { views: [], - amount: initRenderAmount + amount: initRenderAmount, + computing: false } } \ No newline at end of file diff --git a/packages/rath-client/src/store/discovery/mainStore.ts b/packages/rath-client/src/store/discovery/mainStore.ts index 174c7b90..5ffecf80 100644 --- a/packages/rath-client/src/store/discovery/mainStore.ts +++ b/packages/rath-client/src/store/discovery/mainStore.ts @@ -1,5 +1,5 @@ import produce from "immer"; -import { makeAutoObservable, observable, runInAction } from "mobx"; +import { makeAutoObservable, observable, reaction, runInAction } from "mobx"; import { IPattern } from "../../dev"; import { IFieldMeta, IResizeMode } from "../../interfaces"; import { distVis } from "../../queries/distVis"; @@ -18,7 +18,6 @@ export class DiscoveryMainStore { public featViews: IAssoViews; public filterViews: IAssoViews; private dataSourceStore: DataSourceStore; - public computing: boolean = false; public mainView: IPattern | null = null; public compareView: IPattern | null = null; public showMiniFloatView: boolean = false; @@ -29,6 +28,7 @@ export class DiscoveryMainStore { featViews: true, filterViews: true } + private reactions: any[] = []; constructor (dataSourceStore: DataSourceStore) { this.dataSourceStore = dataSourceStore; this.mainVizSetting = { @@ -47,6 +47,26 @@ export class DiscoveryMainStore { this.pattViews = makeInitAssoViews(RENDER_BATCH_SIZE); this.featViews = makeInitAssoViews(RENDER_BATCH_SIZE); this.filterViews = makeInitAssoViews(RENDER_BATCH_SIZE); + this.reactions.push(reaction(() => { + return { + mainView: this.mainView, + dataSource: this.dataSource, + fieldMetas: this.fieldMetas, + autoFeat: this.autoAsso.featViews, + autoPatt: this.autoAsso.pattViews, + autoFilter: this.autoAsso.filterViews + } + }, (props) => { + const { mainView, autoFeat, autoFilter, autoPatt } = props; + if (mainView) { + autoPatt && this.pattAssociate(); + !autoPatt && this.initRenderViews('pattViews') + autoFeat && this.featAssociate(); + !autoPatt && this.initRenderViews('featViews'); + autoFilter && this.filterAssociate(); + !autoFilter && this.initRenderViews('filterViews'); + } + })) makeAutoObservable(this, { pattViews: observable.shallow, @@ -62,6 +82,9 @@ export class DiscoveryMainStore { public updateSettings (skey: keyof ISetting, value: any) { this.settings[skey] = value; } + public initRenderViews (akey: IRenderViewKey) { + this[akey] = makeInitAssoViews(); + } public updateAutoAssoConfig (akey: IRenderViewKey, value: boolean) { this.autoAsso[akey] = value; } @@ -73,6 +96,9 @@ export class DiscoveryMainStore { public setShowMiniFloatView (show: boolean) { this.showMiniFloatView = show; } + public hasMainView () { + return this.mainView !== null + } public get dataSource () { return this.dataSourceStore.cleanedData; } @@ -137,7 +163,7 @@ export class DiscoveryMainStore { this.changeRenderAmount(stateKey, safeSize) } public async featAssociate () { - this.computing = true; + this.featViews.computing = true const { fieldMetas, dataSource, mainView } = this; try { const res = await footmanEngineService({ @@ -149,15 +175,15 @@ export class DiscoveryMainStore { runInAction(() => { this.featViews.views = res; this.featViews.amount = RENDER_BATCH_SIZE; - this.computing = false; + this.featViews.computing = false; }) } catch (error) { console.error(error); - this.computing = false; + this.featViews.computing = false; } } public async pattAssociate () { - this.computing = true; + this.pattViews.computing = true const { fieldMetas, dataSource, mainView } = this; try { const res = await footmanEngineService({ @@ -169,15 +195,15 @@ export class DiscoveryMainStore { runInAction(() => { this.pattViews.views = res; this.pattViews.amount = RENDER_BATCH_SIZE; - this.computing = false; + this.pattViews.computing = false; }) } catch (error) { console.error(error); - this.computing = false; + this.pattViews.computing = false; } } public async initAssociate () { - this.computing = false; + this.pattViews.computing = true; const { dataSource, fieldMetas } = this; try { const res = await footmanEngineService({ @@ -186,18 +212,18 @@ export class DiscoveryMainStore { task: 'univar' }, 'local') runInAction(() => { - this.computing = false; + this.pattViews.computing = false; this.pattViews.views = res; this.pattViews.amount = RENDER_BATCH_SIZE; }) } catch (error) { console.error(error); - this.computing = false; + this.pattViews.computing = false; } } public async filterAssociate () { if (this.mainView === null) return; - this.computing = true; + this.filterViews.computing = true; const { fieldMetas, dataSource, mainView } = this; try { const res = await footmanEngineService({ @@ -209,11 +235,11 @@ export class DiscoveryMainStore { runInAction(() => { this.filterViews.views = res; this.filterViews.amount = RENDER_BATCH_SIZE; - this.computing = false; + this.filterViews.computing = false; }) } catch (error) { console.error(error); - this.computing = false; + this.filterViews.computing = false; } } public removeMainViewFilter (filterFieldId: string) { @@ -242,7 +268,7 @@ export class DiscoveryMainStore { public updateMainView (view: IPattern) { this.mainView = view; // this.initAssociate() - this.clearViews(); + // this.clearViews(); } public updateCompareView (view: IPattern) { this.compareView = view; @@ -250,7 +276,7 @@ export class DiscoveryMainStore { } public async explainViewDiff (view1: IPattern, view2: IPattern) { if (this.mainView === null) return; - this.computing = true; + this.featViews.computing = true const { fieldMetas, dataSource } = this; try { const res = await footmanEngineService<{ features: IFieldMeta[] }>({ @@ -273,11 +299,11 @@ export class DiscoveryMainStore { } ] } - this.computing = false; + this.featViews.computing = false; }) } catch (error) { console.error(error); - this.computing = false; + this.featViews.computing = false } } } \ No newline at end of file diff --git a/packages/rath-client/src/workers/engine/core.ts b/packages/rath-client/src/workers/engine/core.ts index 02658b27..ff0b517a 100644 --- a/packages/rath-client/src/workers/engine/core.ts +++ b/packages/rath-client/src/workers/engine/core.ts @@ -393,7 +393,9 @@ export class RathEngine extends VIEngine { impurity: totalEntLoss }) } + // let ii = 0; for (let space of viewSpaces) { + // console.log(ii++, viewSpaces.length, ii / viewSpaces.length) const { dimensions, measures } = space; let dropSpace = false; const localCuboid = await context.cube.getCuboid(dimensions); From a77aab7c89fe0cb8323cf7896b6677c6188e6af4 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Fri, 29 Jul 2022 17:01:56 +0800 Subject: [PATCH 12/15] feat: auto analysis progress indicator --- .../megaAutomation/computationProgress.tsx | 42 +++++++++++++++++++ .../src/pages/megaAutomation/index.tsx | 7 ++-- packages/rath-client/src/utils/storage.ts | 22 +++++++++- .../rath-client/src/workers/engine/core.ts | 11 +++-- .../rath-client/src/workers/engine/utils.ts | 19 +++++++++ 5 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 packages/rath-client/src/pages/megaAutomation/computationProgress.tsx diff --git a/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx b/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx new file mode 100644 index 00000000..0414f2fe --- /dev/null +++ b/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx @@ -0,0 +1,42 @@ +import { ProgressIndicator } from 'office-ui-fabric-react'; +import React, { useEffect, useState } from 'react'; +import { getStateInStorage } from '../../workers/engine/utils'; +import intl from 'react-intl-universal'; + +interface CPProps { + computing: boolean; +} +const ComputationProgress: React.FC = props => { + const { computing } = props; + + const [pn, setPn] = useState(0); + + useEffect(() => { + let int = -1; + if (computing) { + int = window.setInterval(() => { + getStateInStorage('explore_progress').then((v) => { + if (typeof v === 'number') { + setPn(v) + } + }) + }, 1000) + } + return () => { + if (int !== -1) { + clearInterval(int) + } + } + }, [computing]) + + return
+ { + computing && 0 ? (Math.round(pn * 100) + '%') : ''}`} + percentComplete={pn > 0 ? pn : undefined} + /> + } +
+} + +export default ComputationProgress; diff --git a/packages/rath-client/src/pages/megaAutomation/index.tsx b/packages/rath-client/src/pages/megaAutomation/index.tsx index 4188e014..37d554ad 100644 --- a/packages/rath-client/src/pages/megaAutomation/index.tsx +++ b/packages/rath-client/src/pages/megaAutomation/index.tsx @@ -4,7 +4,7 @@ import { Divider, Pagination } from '@material-ui/core'; import styled from 'styled-components'; import intl from 'react-intl-universal' import { runInAction } from 'mobx'; -import { DefaultButton, Stack, ProgressIndicator, CommandBarButton, IconButton, Toggle, Dropdown, IDropdownOption } from 'office-ui-fabric-react'; +import { DefaultButton, Stack, CommandBarButton, IconButton, Toggle, Dropdown, IDropdownOption } from 'office-ui-fabric-react'; import { useGlobalStore } from '../../store'; import BaseChart from '../../visBuilder/vegaBase'; @@ -19,6 +19,7 @@ import FieldContainer from './vizOperation/fieldContainer'; import { IResizeMode } from '../../interfaces'; import ResizeContainer from './resizeContainer'; import Narrative from './narrative'; +import ComputationProgress from './computationProgress'; const MARGIN_LEFT = { marginLeft: '1em' }; @@ -119,9 +120,7 @@ const LTSPage: React.FC = () => { onClick={downloadResults} /> -
- { computing && } -
+ {intl.get('lts.title')}

{intl.get('lts.hintMain')}

diff --git a/packages/rath-client/src/utils/storage.ts b/packages/rath-client/src/utils/storage.ts index 48babf33..6817f33b 100644 --- a/packages/rath-client/src/utils/storage.ts +++ b/packages/rath-client/src/utils/storage.ts @@ -7,7 +7,8 @@ export const STORAGE_INSTANCE = 'rath_storage_instance' const STORAGES = { DATASOURCE: 'datasource', WORKSPACE: 'workspace', - META: 'meta' + META: 'meta', + STATE: 'state' } export interface IDBMeta { @@ -144,4 +145,21 @@ export async function setDataStorage(name: string, fields: IMuteFieldBase[], dat storeName: STORAGES.DATASOURCE }); storages.setItem(name, dataString); -} \ No newline at end of file +} + +// export async function setStateInStorage(key: string, value: any) { +// // const state = localforage.createInstance({ +// // name: STORAGE_INSTANCE, +// // storeName: STORAGES.STATE +// // }); +// // await state.setItem(key, value) +// } + +// export async function getStateInStorage(key: string) { +// const state = localforage.createInstance({ +// name: STORAGE_INSTANCE, +// storeName: STORAGES.STATE +// }); +// const val = await state.getItem(key) +// return val; +// } \ No newline at end of file diff --git a/packages/rath-client/src/workers/engine/core.ts b/packages/rath-client/src/workers/engine/core.ts index ff0b517a..acfc1d3c 100644 --- a/packages/rath-client/src/workers/engine/core.ts +++ b/packages/rath-client/src/workers/engine/core.ts @@ -1,8 +1,10 @@ + import { IInsightSpace, InsightFlow, Cube, ViewSpace, } from 'visual-insights' import { IRow } from '../../interfaces'; import { IVizSpace } from '../../store/exploreStore'; import { isSetEqual } from '../../utils'; -import { intersect } from './utils'; +// import { setStateInStorage } from '../../utils/storage'; +import { intersect, setStateInStorage } from './utils'; const { VIEngine, DataGraph } = InsightFlow @@ -19,7 +21,6 @@ export function entropyAcc (fl: number[]) { return -ent; } - class CustomDataGraph extends DataGraph { // public clusterDGraph(dataSource: IRow[], CORRELATION_THRESHOLD?: number) { @@ -393,9 +394,11 @@ export class RathEngine extends VIEngine { impurity: totalEntLoss }) } - // let ii = 0; + let ii = 0; for (let space of viewSpaces) { - // console.log(ii++, viewSpaces.length, ii / viewSpaces.length) + ii++; + // FIXME: throtte + ii % 10 === 0 && setStateInStorage('explore_progress', ii / viewSpaces.length) const { dimensions, measures } = space; let dropSpace = false; const localCuboid = await context.cube.getCuboid(dimensions); diff --git a/packages/rath-client/src/workers/engine/utils.ts b/packages/rath-client/src/workers/engine/utils.ts index ce11937e..e5450738 100644 --- a/packages/rath-client/src/workers/engine/utils.ts +++ b/packages/rath-client/src/workers/engine/utils.ts @@ -1,3 +1,5 @@ +import localforage from "localforage"; + export function intersect (A: string[], B: string[]) { const bset = new Set(B); if (A.length === 0 || B.length === 0) return true; @@ -5,4 +7,21 @@ export function intersect (A: string[], B: string[]) { if (bset.has(a)) return true } return false; +} + +export async function setStateInStorage(key: string, value: any) { + const state = localforage.createInstance({ + name: 'STORAGE_INSTANCE', + storeName: 'STORAGES.STATE' + }); + await state.setItem(key, value) +} + +export async function getStateInStorage(key: string) { + const state = localforage.createInstance({ + name: 'STORAGE_INSTANCE', + storeName: 'STORAGES.STATE' + }); + const val = await state.getItem(key) + return val; } \ No newline at end of file From 762786bae802bebe20dedf4084656926d1140dcd Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Thu, 4 Aug 2022 21:12:19 +0800 Subject: [PATCH 13/15] fix: bugs --- packages/rath-client/src/dev/index.ts | 60 ++++------------- packages/rath-client/src/dev/utils.ts | 64 +++++++++++++------ .../src/pages/dataSource/utils/tools.ts | 2 +- .../megaAutomation/computationProgress.tsx | 6 +- .../focusZone/miniFloatCanvas.tsx | 2 +- .../src/store/discovery/mainStore.ts | 38 ++++++----- .../src/workers/footman/service.ts | 2 +- 7 files changed, 85 insertions(+), 89 deletions(-) diff --git a/packages/rath-client/src/dev/index.ts b/packages/rath-client/src/dev/index.ts index 9581f11a..a24844e8 100644 --- a/packages/rath-client/src/dev/index.ts +++ b/packages/rath-client/src/dev/index.ts @@ -217,31 +217,33 @@ export class NextVICore { return matrix } - public fewatureSelectionForSecondPatternWithSpecifiedViews (patt1: [IFieldMeta, IFieldMeta], patt2: [IFieldMeta, IFieldMeta]): { + public fewatureSelectionForSecondPatternWithSpecifiedViews (patt1: IPattern, patt2: IPattern): { features: IFieldMeta[]; score: number; } | null { const { dataSource } = this; + const view1Meas = patt1.fields.filter(f => f.analyticType === 'measure'); + const view2Meas = patt2.fields.filter(f => f.analyticType === 'measure'); + const dataView1 = applyFilter(dataSource, patt1.filters); + const dataView2 = applyFilter(dataSource, patt2.filters); + if (patt1.fields.length !== 2 || patt2.fields.length !== 2) throw new Error('View size Not supported yet!') const dimensions = this.fields.filter(f => f.analyticType === 'dimension'); - // const measures = this.fields.filter(f => f.analyticType === 'measure'); - let patt1Points = dataSource.map(row => [row[patt1[0].fid], row[patt1[1].fid]]) as [number, number][]; - let patt2Points = dataSource.map(row => [row[patt2[0].fid], row[patt2[1].fid]]) as [number, number][]; + let patt1Dist: [number, number][] = dataView1.map(row => view1Meas.map(m => row[m.fid])) as [number, number][]; + let patt2Dist: [number, number][] = dataView2.map(row => view2Meas.map(m => row[m.fid])) as [number, number][]; + let bestDScore = 0; let bestDIndex = -1; for (let k = 0; k < dimensions.length; k++) { - const t = dataSource.map(row => row[dimensions[k].fid]) - const dimScore = incSim(t, patt1Points, patt2Points) + const T1 = dataView1.map(row => row[dimensions[k].fid]) + const T2 = dataView2.map(row => row[dimensions[k].fid]); + const dimScore = incSim(T1, patt1Dist, T2, patt2Dist, dataSource.length) + console.log(dimensions[k].fid, dimScore) if (dimScore > bestDScore) { bestDScore = dimScore; bestDIndex = k; } } if (bestDIndex > -1) { - // console.log( - // patt1.map(f => f.fid), - // patt2.map(f => f.fid), - // dimensions[bestDIndex].fid, - // bestDScore); return { features: [dimensions[bestDIndex]], score: bestDScore @@ -300,40 +302,4 @@ export class NextVICore { ans.sort((a, b) => b.imp - a.imp) return ans; } - - public featureSelectForSecondPattern () { - const { dataSource } = this; - const dimensions = this.fields.filter(f => f.analyticType === 'dimension'); - const measures = this.fields.filter(f => f.analyticType === 'measure'); - const patterns: [IFieldMeta, IFieldMeta][] = []; - for (let i = 0; i < measures.length; i++) { - for (let j = i + 1; j < measures.length; j++) { - patterns.push([measures[i], measures[j]]) - } - } - const patternNum = patterns.length; - for (let i = 0; i < patternNum; i++) { - for (let j = i + 1; j < patternNum; j++) { - let pattX = dataSource.map(row => [row[patterns[i][0].fid], row[patterns[i][1].fid]]) as [number, number][]; - let pattY = dataSource.map(row => [row[patterns[j][0].fid], row[patterns[j][1].fid]]) as [number, number][]; - let bestDScore = 0; - let bestDIndex = -1; - for (let k = 0; k < dimensions.length; k++) { - const t = dataSource.map(row => row[dimensions[k].fid]) - const dimScore = incSim(t, pattX, pattY) - if (dimScore > bestDScore) { - bestDScore = dimScore; - bestDIndex = k; - } - } - if (bestDIndex > -1) { - console.log( - patterns[i].map(f => f.fid), - patterns[j].map(f => f.fid), - dimensions[bestDIndex].fid, - bestDScore); - } - } - } - } } \ No newline at end of file diff --git a/packages/rath-client/src/dev/utils.ts b/packages/rath-client/src/dev/utils.ts index bb0d4a5d..7cbb9499 100644 --- a/packages/rath-client/src/dev/utils.ts +++ b/packages/rath-client/src/dev/utils.ts @@ -288,39 +288,61 @@ export function normalizeScatter (points: [number, number][]) { pbMatrix[i][j] = matrix[i][j] / points.length; } } - // console.log(pbMatrix) return pbMatrix } -interface IPatternPair { - X: [number, number][]; - Y: [number, number][]; -} -export function incSim (T: string[], pointsX: [number, number][], pointsY: [number, number][]) { +export function incSim (TX: string[], pointsX: [number, number][], TY: string[], pointsY: [number, number][], TSize: number) { const S = l2Dis2(normalizeScatter(pointsX), normalizeScatter(pointsY)); - let groups: Map = new Map() - for (let i = 0; i < T.length; i++) { - if (!groups.has(T[i])) { - const pair: IPatternPair = { - X: [], - Y: [] - }; - groups.set(T[i], pair) + let groupsX: Map = new Map(); + let groupsY: Map = new Map(); + for (let i = 0; i < TX.length; i++) { + if (!groupsX.has(TX[i])) { + groupsX.set(TX[i], []) + } + groupsX.get(TX[i])!.push(pointsX[i]) + } + for (let i = 0; i < TY.length; i++) { + if (!groupsY.has(TY[i])) { + groupsY.set(TY[i], []) } - groups.get(T[i])?.X.push(pointsX[i]) - groups.get(T[i])?.Y.push(pointsY[i]) + groupsY.get(TY[i])!.push(pointsY[i]) } + let condS = 0; - for (let [, pair] of groups.entries()) { - let p = pair.X.length / pointsX.length; + for (let [t, vecX] of groupsX.entries()) { + let p = vecX.length / TSize; if (p === 0) continue; - if (pair.X.length < BIN_SIZE ** 2) { + if (vecX.length < BIN_SIZE ** 2) { + condS += p; + continue; + } + if (!groupsY.has(t)) { condS += p; continue; } - // let p = 1 / groups.size - condS += (p * l2Dis2(normalizeScatter(pair.X), normalizeScatter(pair.Y))); + const vecY = groupsY.get(t)!; + condS += (p * l2Dis2(normalizeScatter(vecX), normalizeScatter(vecY))) } + for (let [t, vecY] of groupsY.entries()) { + let p = vecY.length / TSize; + if (p === 0) continue; + if (!groupsX.has(t)) { + condS += p; + continue; + } + } + // for (let [, pair] of groups.entries()) { + // let p = pair.X.length / pointsX.length; + // if (p === 0) continue; + // if (pair.X.length < BIN_SIZE ** 2) { + // condS += p; + // continue; + // } + // if () + // // let p = 1 / groups.size + // console.log(pair.X, pair.Y) + // condS += (p * l2Dis2(normalizeScatter(pair.X), normalizeScatter(pair.Y))); + // } return S - condS; } diff --git a/packages/rath-client/src/pages/dataSource/utils/tools.ts b/packages/rath-client/src/pages/dataSource/utils/tools.ts index 834787ee..2d52cf38 100644 --- a/packages/rath-client/src/pages/dataSource/utils/tools.ts +++ b/packages/rath-client/src/pages/dataSource/utils/tools.ts @@ -21,7 +21,7 @@ import { formatTimeField } from "../../../utils/transform"; */ function formatColKeys(colKeys: string[]): string[] { return colKeys.map((col, colIndex) => { - return `col_${colIndex}_${Math.round(Math.random() * 100)}` + return `col_${colIndex}` }) } diff --git a/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx b/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx index 0414f2fe..2a4b37b5 100644 --- a/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx +++ b/packages/rath-client/src/pages/megaAutomation/computationProgress.tsx @@ -1,6 +1,6 @@ import { ProgressIndicator } from 'office-ui-fabric-react'; import React, { useEffect, useState } from 'react'; -import { getStateInStorage } from '../../workers/engine/utils'; +import { getStateInStorage, setStateInStorage } from '../../workers/engine/utils'; import intl from 'react-intl-universal'; interface CPProps { @@ -11,6 +11,10 @@ const ComputationProgress: React.FC = props => { const [pn, setPn] = useState(0); + useEffect(() => { + setStateInStorage('explore_progress', 0) + }, []) + useEffect(() => { let int = -1; if (computing) { diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx index 74760e7a..4f1d64ac 100644 --- a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx @@ -16,7 +16,7 @@ const FloatContainer = styled.div<{hide: boolean}>` position: fixed; right: 1px; top: 30%; - z-index: 99; + z-index: 999; padding: ${props => props.hide ? '5px' : '1em'}; background-color: #fff; box-shadow: 0 10px 8px rgba(0, 0, 0, 0.05), 0 4px 3px rgba(0, 0, 0, 0.01); diff --git a/packages/rath-client/src/store/discovery/mainStore.ts b/packages/rath-client/src/store/discovery/mainStore.ts index 5ffecf80..bcbc89cd 100644 --- a/packages/rath-client/src/store/discovery/mainStore.ts +++ b/packages/rath-client/src/store/discovery/mainStore.ts @@ -282,25 +282,29 @@ export class DiscoveryMainStore { const res = await footmanEngineService<{ features: IFieldMeta[] }>({ dataSource, fields: fieldMetas, - task: 'filterSelection', + task: 'comparison', props: [view1, view2] }, 'local') - runInAction(() => { - if (this.mainView) { - this.clearViews(); - this.featViews.views = [ - { - ...this.mainView, - fields: [...this.mainView!.fields, ...res.features] - }, - { - ...this.mainView, - fields: [...this.mainView!.fields, ...res.features] - } - ] - } - this.featViews.computing = false; - }) + if (res === null) { + this.clearViews(); + } else { + runInAction(() => { + if (this.mainView) { + this.clearViews(); + this.featViews.views = [ + { + ...this.mainView, + fields: [...this.mainView!.fields, ...res.features] + }, + { + ...this.mainView, + fields: [...this.mainView!.fields, ...res.features] + } + ] + } + this.featViews.computing = false; + }) + } } catch (error) { console.error(error); this.featViews.computing = false diff --git a/packages/rath-client/src/workers/footman/service.ts b/packages/rath-client/src/workers/footman/service.ts index 3a07419e..fda7bcc0 100644 --- a/packages/rath-client/src/workers/footman/service.ts +++ b/packages/rath-client/src/workers/footman/service.ts @@ -41,7 +41,7 @@ function featureSelection (dataSource: IRow[], fields: IFieldMeta[], props: IPat function featureForComparison (dataSouce: IRow[], fields: IFieldMeta[], props: [IPattern, IPattern]) { const core = new NextVICore(dataSouce, fields); - const ans = core.fewatureSelectionForSecondPatternWithSpecifiedViews(props[0].fields.slice(0, 2) as [IFieldMeta, IFieldMeta], props[1].fields.slice(0, 2) as [IFieldMeta, IFieldMeta]) + const ans = core.fewatureSelectionForSecondPatternWithSpecifiedViews(props[0], props[1]) return ans; } From 037c2a24cb49d667d34c64d4c7b24fab7213fe39 Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Sat, 6 Aug 2022 18:39:12 +0800 Subject: [PATCH 14/15] feat: airtable connection --- packages/rath-client/.prettierrc.json | 1 + packages/rath-client/package.json | 4 +- .../rath-client/public/locales/zh-CN.json | 4 +- packages/rath-client/src/global.ts | 3 +- .../src/pages/dataSource/config.ts | 18 +++-- .../dataSource/selection/airtable/index.tsx | 70 +++++++++++++++++++ .../dataSource/selection/airtable/utils.ts | 38 ++++++++++ .../src/pages/dataSource/selection/index.tsx | 2 + .../src/pages/dataSource/utils/index.ts | 4 +- .../src/pages/dataSource/utils/tools.ts | 2 +- yarn.lock | 38 ++++++++++ 11 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 packages/rath-client/.prettierrc.json create mode 100644 packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx create mode 100644 packages/rath-client/src/pages/dataSource/selection/airtable/utils.ts diff --git a/packages/rath-client/.prettierrc.json b/packages/rath-client/.prettierrc.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/packages/rath-client/.prettierrc.json @@ -0,0 +1 @@ +{} diff --git a/packages/rath-client/package.json b/packages/rath-client/package.json index 2f0c12d3..fe088ddf 100644 --- a/packages/rath-client/package.json +++ b/packages/rath-client/package.json @@ -5,7 +5,7 @@ "homepage": "./", "prettier": { "tabWidth": 4, - "printWidth": 140, + "printWidth": 120, "singleQuote": true }, "dependencies": { @@ -16,6 +16,7 @@ "@material-ui/core": "^5.0.0-beta.5", "@uifabric/icons": "^7.5.17", "@uifabric/react-hooks": "^7.13.9", + "airtable": "^0.11.4", "ali-react-table": "^2.3.1", "chroma-js": "^2.0.6", "cube-core": "^2.13.0", @@ -61,6 +62,7 @@ "@types/react-dom": "^17.0.1", "@types/styled-components": "^5.1.7", "customize-cra": "^1.0.0", + "prettier": "2.7.1", "source-map-explorer": "^2.5.2", "typescript": "^4.7.4" }, diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index e09cb643..5d949c93 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -138,7 +138,7 @@ "tip": "记得在开始分析前先调整数据的清洗策略并配置字段的属性", "recordCount": "记录数 {count}", "upload": { - "title": "上传你的数据集,根据需求调整以下配置", + "title": "连接你的数据集,根据需求调整以下配置", "fileTypes": "支持csv, json文件", "uniqueIdIssue": "添加唯一标识(字段是中文字符推荐使用)", "sampling": "数据采样", @@ -226,7 +226,7 @@ "pageNo": "报表序号: 当前 {current} 总 {total}" }, "explainer": { - "notReadyTip": "如果你看到这段文字,说明你还没有在数据页面上传你的数据或者还没有点击数据集页面的获取洞察按钮(它会产生一个当前页面需要使用的计算结果)", + "notReadyTip": "如果你看到这段文字,说明你还没有在数据页面连接你的数据或者还没有点击数据集页面的获取洞察按钮(它会产生一个当前页面需要使用的计算结果)", "insightButton": "获取洞察", "sigThreshold": "显著性阈值", "desc": "总共有 {num} 个视图中的洞察显著性不低于 {threshold} %", diff --git a/packages/rath-client/src/global.ts b/packages/rath-client/src/global.ts index d64782e2..05e74ac9 100644 --- a/packages/rath-client/src/global.ts +++ b/packages/rath-client/src/global.ts @@ -21,7 +21,8 @@ export enum IDataSourceType { MYSQL = 'mysql', DEMO = 'demo', CLICKHOUSE = 'clickhouse', - LOCAL = 'local' + LOCAL = 'local', + AIRTABLE = 'airtable' } export const globalRef: { diff --git a/packages/rath-client/src/pages/dataSource/config.ts b/packages/rath-client/src/pages/dataSource/config.ts index bef2bbc3..7d403da2 100644 --- a/packages/rath-client/src/pages/dataSource/config.ts +++ b/packages/rath-client/src/pages/dataSource/config.ts @@ -33,17 +33,23 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp text: restfulText, iconProps: { iconName: "Cloud" }, }, - { - key: IDataSourceType.MYSQL, - text: mysqlText, - iconProps: { iconName: "LinkedDatabase" }, - disabled: true, - }, + // { + // key: IDataSourceType.MYSQL, + // text: mysqlText, + // iconProps: { iconName: "LinkedDatabase" }, + // disabled: true, + // }, { key: IDataSourceType.CLICKHOUSE, text: clickHouseText, iconProps: { iconName: "TripleColumn" }, disabled: false, + }, + { + key: IDataSourceType.AIRTABLE, + text: 'AirTable', + iconProps: { iconName: 'Table' }, + disabled: false } ]; }, [fileText, restfulText, mysqlText, demoText, localText]); diff --git a/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx b/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx new file mode 100644 index 00000000..3fd096c8 --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx @@ -0,0 +1,70 @@ +import { DefaultButton, PrimaryButton, Stack, TextField } from 'office-ui-fabric-react'; +import React, { useCallback, useState } from 'react'; +import { IMuteFieldBase, IRow } from '../../../../interfaces'; +import { transformRawDataService } from '../../utils'; +import { fetchAllRecordsFromAirTable } from './utils'; +import intl from 'react-intl-universal' + +interface AirTableSourceProps { + onClose: () => void; + onStartLoading: () => void; + onLoadingFailed: (err: any) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[]) => void; +} +const AirTableSource: React.FC = (props) => { + const { onClose, onDataLoaded, onLoadingFailed, onStartLoading } = props; + const [endPoint, setEndPoint] = useState(''); + const [apiKey, setAPIKey] = useState(''); + const [tableID, setTableID] = useState(''); + const [tableName, setTableName] = useState(''); + const [viewName, setViewName] = useState(''); + + const fetchData = useCallback(() => { + onStartLoading(); + fetchAllRecordsFromAirTable({ + endPoint, + apiKey, + tableID, + tableName, + viewName + }) + .then((data) => transformRawDataService(data)) + .then((ds) => { + onDataLoaded(ds.fields, ds.dataSource); + onClose(); + }) + .catch(onLoadingFailed); + }, [onDataLoaded, onClose, onLoadingFailed, onStartLoading, endPoint, apiKey, tableID, tableName, viewName]); + return ( +
+ + { setEndPoint(`${value}`) }} + value={endPoint} + /> + { setAPIKey(`${value}`) }} + value={apiKey} + /> + { setTableID(`${value}`) }} + value={tableID} + /> + { setTableName(`${value}`) }} + value={tableName} + /> + { setViewName(`${value}`) }} + value={viewName} + /> + + { intl.get('dataSource.importData.load') } + Cancel + + +
+ ); +}; + +export default AirTableSource; diff --git a/packages/rath-client/src/pages/dataSource/selection/airtable/utils.ts b/packages/rath-client/src/pages/dataSource/selection/airtable/utils.ts new file mode 100644 index 00000000..0b68345e --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/airtable/utils.ts @@ -0,0 +1,38 @@ +// interface +import Airtable from "airtable"; +import { AirtableBase } from "airtable/lib/airtable_base"; +import { IRow } from "visual-insights"; + +interface IAirtableAPIBase { + tableName: string; + viewName: string; + endPoint: string; + apiKey: string; + tableID: string; +} + +function selectAllPromise (base: AirtableBase, tableName: string, viewName: string) { + return new Promise((resolve, reject) => { + const rows: IRow[] = [] + base(tableName).select({ + view: viewName + }).eachPage((records, fetchNextPage) => { + for (let rec of records) { + rows.push({ ...rec.fields }) + } + fetchNextPage() + }, (err) => { + if (err) { reject(err); return; } + resolve(rows) + }) + }) +} +export async function fetchAllRecordsFromAirTable (props: IAirtableAPIBase): Promise { + const at = new Airtable({ + endpointUrl: props.endPoint, + apiKey: props.apiKey, + }) + const base = at.base(props.tableID) + const res = await selectAllPromise(base, props.tableName, props.viewName); + return res; +} \ No newline at end of file diff --git a/packages/rath-client/src/pages/dataSource/selection/index.tsx b/packages/rath-client/src/pages/dataSource/selection/index.tsx index f9581b3b..3428b853 100644 --- a/packages/rath-client/src/pages/dataSource/selection/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/index.tsx @@ -12,6 +12,7 @@ import ClickHouseData from './clickhouse'; import { IMuteFieldBase, IRow } from '../../../interfaces'; import Local from './local'; import DataLoadingStatus from '../dataLoadingStatus'; +import AirTableSource from './airtable'; interface SelectionProps { show: boolean; @@ -55,6 +56,7 @@ const Selection: React.FC = props => { {dataSourceType === IDataSourceType.CLICKHOUSE && } {dataSourceType === IDataSourceType.RESTFUL && } {dataSourceType === IDataSourceType.LOCAL && } + {dataSourceType === IDataSourceType.AIRTABLE && }
); diff --git a/packages/rath-client/src/pages/dataSource/utils/index.ts b/packages/rath-client/src/pages/dataSource/utils/index.ts index 8b739a91..33e01075 100644 --- a/packages/rath-client/src/pages/dataSource/utils/index.ts +++ b/packages/rath-client/src/pages/dataSource/utils/index.ts @@ -37,7 +37,7 @@ export const useSampleOptions = function () { return options; } -async function transformFileDataService (rawData: IRow[]): Promise<{ +export async function transformRawDataService (rawData: IRow[]): Promise<{ fields: IMuteFieldBase[]; dataSource: IRow[] }> { @@ -107,7 +107,7 @@ export async function loadDataFile(props: LoadDataFileProps): Promise<{ } else { throw new Error(`unsupported file type=${file.type} `) } - const dataset = await transformFileDataService(rawData); + const dataset = await transformRawDataService(rawData); return dataset } diff --git a/packages/rath-client/src/pages/dataSource/utils/tools.ts b/packages/rath-client/src/pages/dataSource/utils/tools.ts index 2d52cf38..43895ab0 100644 --- a/packages/rath-client/src/pages/dataSource/utils/tools.ts +++ b/packages/rath-client/src/pages/dataSource/utils/tools.ts @@ -21,7 +21,7 @@ import { formatTimeField } from "../../../utils/transform"; */ function formatColKeys(colKeys: string[]): string[] { return colKeys.map((col, colIndex) => { - return `col_${colIndex}` + return `c_${colIndex}` }) } diff --git a/yarn.lock b/yarn.lock index 37a057c1..b3dd988c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2963,6 +2963,11 @@ resolved "https://registry.nlark.com/@types/node/download/@types/node-15.12.5.tgz?cache=0&sync_timestamp=1624763053899&other_urls=https%3A%2F%2Fregistry.nlark.com%2F%40types%2Fnode%2Fdownload%2F%40types%2Fnode-15.12.5.tgz#9a78318a45d75c9523d2396131bd3cca54b2d185" integrity sha1-mngxikXXXJUj0jlhMb08ylSy0YU= +"@types/node@>=8.0.0 <15": + version "14.18.23" + resolved "https://registry.npmmirror.com/@types/node/-/node-14.18.23.tgz#70f5f20b0b1b38f696848c1d3647bb95694e615e" + integrity sha512-MhbCWN18R4GhO8ewQWAFK4TGQdBpXWByukz7cWyJmXhvRuCIaM/oWytGPqVmDzgEnnaIc9ss6HbU5mUi+vyZPA== + "@types/node@^10.0.3": version "10.17.60" resolved "https://registry.nlark.com/@types/node/download/@types/node-10.17.60.tgz?cache=0&sync_timestamp=1624763053899&other_urls=https%3A%2F%2Fregistry.nlark.com%2F%40types%2Fnode%2Fdownload%2F%40types%2Fnode-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b" @@ -3697,6 +3702,18 @@ abab@^2.0.3, abab@^2.0.5: resolved "https://registry.npm.taobao.org/abab/download/abab-2.0.5.tgz?cache=0&sync_timestamp=1599850271460&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fabab%2Fdownload%2Fabab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" integrity sha1-wLZ4+zLWD8EhnHhNaoJv44Wut5o= +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.npmmirror.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + +abortcontroller-polyfill@^1.4.0: + version "1.7.3" + resolved "https://registry.npmmirror.com/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.3.tgz#1b5b487bd6436b5b764fd52a612509702c3144b5" + integrity sha512-zetDJxd89y3X99Kvo4qFx8GKlt6GsvN3UcRZHwU6iFA/0KiOmhkTVhe8oRoTBiTVPZu09x3vCra47+w8Yz1+2Q== + accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: version "1.3.7" resolved "https://registry.npm.taobao.org/accepts/download/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" @@ -3790,6 +3807,17 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" +airtable@^0.11.4: + version "0.11.4" + resolved "https://registry.npmmirror.com/airtable/-/airtable-0.11.4.tgz#1f8bef44c82a185f05655fc5e9c9563b1ee775ec" + integrity sha512-y8sEi/sMEdgtgvv0V8AhJyFQB9za+6qGIxllt60Ey5ELBgu+bNEc1hjY2aqzp9C38p0ORfaeRS5RSFWlcq11Aw== + dependencies: + "@types/node" ">=8.0.0 <15" + abort-controller "^3.0.0" + abortcontroller-polyfill "^1.4.0" + lodash "^4.17.21" + node-fetch "^2.6.7" + ajv-errors@^1.0.0: version "1.0.1" resolved "https://registry.nlark.com/ajv-errors/download/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d" @@ -7340,6 +7368,11 @@ etag@~1.8.1: resolved "https://registry.nlark.com/etag/download/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.npmmirror.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter3@^4.0.0: version "4.0.7" resolved "https://registry.npm.taobao.org/eventemitter3/download/eventemitter3-4.0.7.tgz?cache=0&sync_timestamp=1598517819668&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Feventemitter3%2Fdownload%2Feventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" @@ -12548,6 +12581,11 @@ prepend-http@^1.0.0: resolved "https://registry.nlark.com/prepend-http/download/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc" integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw= +prettier@2.7.1: + version "2.7.1" + resolved "https://registry.npmmirror.com/prettier/-/prettier-2.7.1.tgz#e235806850d057f97bb08368a4f7d899f7760c64" + integrity sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g== + pretty-bytes@^5.3.0: version "5.6.0" resolved "https://registry.nlark.com/pretty-bytes/download/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" From d58bd18bf259c8294bf113dff3ce349cc709946c Mon Sep 17 00:00:00 2001 From: Hao Chen <270001151@qq.com> Date: Tue, 9 Aug 2022 20:50:32 +0800 Subject: [PATCH 15/15] feat: fix semi-automation --- packages/rath-client/package.json | 1 + packages/rath-client/src/dev/extend.ts | 6 - packages/rath-client/src/dev/index.ts | 305 ------------ packages/rath-client/src/dev/utils.ts | 457 ------------------ .../semiAutomation/focusZone/mainCanvas.tsx | 2 +- .../focusZone/miniFloatCanvas.tsx | 2 +- .../pages/semiAutomation/mainCanvas/index.tsx | 2 +- .../src/pages/semiAutomation/utils.ts | 2 +- packages/rath-client/src/queries/distVis.ts | 2 +- .../rath-client/src/queries/labdistVis.ts | 4 +- .../src/store/discovery/localTypes.ts | 2 +- .../src/store/discovery/mainStore.ts | 2 +- .../src/workers/fieldsSummary.worker.js | 2 +- .../src/workers/footman/service.ts | 2 +- yarn.lock | 5 + 15 files changed, 17 insertions(+), 779 deletions(-) delete mode 100644 packages/rath-client/src/dev/extend.ts delete mode 100644 packages/rath-client/src/dev/index.ts delete mode 100644 packages/rath-client/src/dev/utils.ts diff --git a/packages/rath-client/package.json b/packages/rath-client/package.json index fe088ddf..7356d434 100644 --- a/packages/rath-client/package.json +++ b/packages/rath-client/package.json @@ -12,6 +12,7 @@ "@emotion/react": "^11.4.1", "@emotion/styled": "^11.3.0", "@kanaries/graphic-walker": "0.1.0", + "@kanaries/loa": "^0.0.2", "@kanaries/web-data-loader": "0.1.7", "@material-ui/core": "^5.0.0-beta.5", "@uifabric/icons": "^7.5.17", diff --git a/packages/rath-client/src/dev/extend.ts b/packages/rath-client/src/dev/extend.ts deleted file mode 100644 index 3c9b8b82..00000000 --- a/packages/rath-client/src/dev/extend.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { IFieldMeta, IRow } from "../interfaces"; - -export function temporalExtend (dataSource: IRow[], fields: IFieldMeta[]) { - const temporalFields = fields.filter(f => f.semanticType === 'temporal'); - -} \ No newline at end of file diff --git a/packages/rath-client/src/dev/index.ts b/packages/rath-client/src/dev/index.ts deleted file mode 100644 index a24844e8..00000000 --- a/packages/rath-client/src/dev/index.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { IRow, Statistics } from "visual-insights"; -import { IFieldMeta } from "../interfaces"; -import { getRange } from "../utils"; -import { bin, binMapShareRange, binShareRange, generalMatMic, generalMic, incSim, l1Dis2, mic, normalizeScatter, rangeNormilize } from "./utils"; - -const { entropy, getCombination } = Statistics; - -export interface IFilter { - field: IFieldMeta; - values: any[]; -} -export interface IPattern { - fields: IFieldMeta[]; - imp: number; - filters?: IFilter[] - // rows: IRow[]; -} - -export function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { - if (!filters || filters.length === 0) return dataSource; - return dataSource.filter(row => { - return filters.every(f => f.values.includes(row[f.field.fid])) - }) -} -// FIXME: 洞察计算时,没有考虑把数据集做filter -// TODO: 要能清洗的看出来,filter前后的比对 -// TODO: 每次要计算当前层级的信息增益。 -export class NextVICore { - public BIN_SIZE = 16; - public dataSource: IRow[] = []; - public fields: IFieldMeta[] = []; - public patterns: IPattern[] = []; - constructor (dataSource: IRow[], fields: IFieldMeta[]) { - this.dataSource = dataSource; - this.fields = fields; - } - public init(dataSource: IRow[], fields: IFieldMeta[]) { - this.dataSource = dataSource; - this.fields = fields; - this.patterns = []; - } - public recommandFilter (view: IPattern) { - const viewDimensions = view.fields.filter(f => f.analyticType === 'dimension'); - const viewMeasures = view.fields.filter(f => f.analyticType === 'measure'); - let viewDimNotInFilters: IFieldMeta[] = [...viewDimensions]; - if (typeof view.filters !== 'undefined') { - viewDimNotInFilters = viewDimensions.filter(vd => !view.filters!.find(vf => vf.field.fid === vd.fid)); - } else { - viewDimNotInFilters = [...viewDimensions] - } - const ans: IPattern[] = []; - if (viewDimNotInFilters.length > 0) { - let ansDimKey = ''; - let dimvs: any[] = []; - let bestScore = 0; - for (let dim of viewDimNotInFilters) { - const dimValues = this.dataSource.map(row => row[dim.fid]); - let meanScore = 0; - viewMeasures.forEach(mea => { - const meaValues = this.dataSource.map(row => row[mea.fid]); - const score = generalMic(dimValues, meaValues); - meanScore += score; - }) - meanScore /= viewMeasures.length; - if (meanScore > bestScore) { - bestScore = meanScore; - ansDimKey = dim.fid; - dimvs = dimValues; - } - } - if (ansDimKey!== '') { - const dimUniques = new Set(dimvs); - for (let v of dimUniques.values()) { - let ansFilters: IFilter[] = []; - const viewFieldsInFilter: IFieldMeta[] = []; // = view.fields.find(f => f.fid === ansDimKey)!; - const viewFieldsNotInFilter: IFieldMeta[] = []; - for (let field of view.fields) { - if (field.fid === ansDimKey) viewFieldsInFilter.push(field); - else { - viewFieldsNotInFilter.push(field) - } - } - if (viewFieldsInFilter.length === 0) continue; - if (view.filters) { - ansFilters.push(...view.filters) - } - // const groups = new Map(); - let score = 0; - for (let mea of viewMeasures) { - // const meaRange = getRange(this.dataSource.map(row => row[mea.fid])) - const values = this.dataSource.map(row => row[mea.fid]); - const meaRange = getRange(values) - const globalDist = rangeNormilize(binShareRange(values, meaRange[0], meaRange[1])); - const subMeaValues: number[] = this.dataSource.filter(row => row[ansDimKey] === v).map(row => row[mea.fid]); - const subDist = rangeNormilize(binMapShareRange(subMeaValues, meaRange[0], meaRange[1])); - let kl = 0; - for (let i = 0; i < globalDist.length; i++) { - if (globalDist[i] > 0 && subDist[i] > 0) { - kl += globalDist[i] * Math.log2(globalDist[i] / subDist[i]) - } - } - // score = Math.max(kl, score) - score += kl * (subMeaValues.length / values.length); - } - score /= viewMeasures.length; - - ansFilters.push({ - field: viewFieldsInFilter[0], - values: [v] - }) - ans.push({ - imp: score, - fields: viewFieldsNotInFilter, - filters: ansFilters - }) - } - } - } - ans.sort((a, b) => b.imp - a.imp) - return ans; - } - public searchPatterns () { - const { dataSource } = this; - const measures = this.fields.filter(f => f.analyticType === 'measure'); - for (let i = 0; i < measures.length; i++) { - const values = dataSource.map(row => row[measures[i].fid]); - // let _max = Math.max(...values); - // let _min = Math.min(...values); - // let step = (_max - _min) / this.BIN_SIZE; - const bins = bin(values); - const pl = rangeNormilize(bins.filter(f => f > 0)) - const ent = entropy(pl); - this.patterns.push({ - fields: [measures[i]], - imp: ent, - // rows: bins.map((v, vi) => ({ - // bin: _min + vi * step, - // count: v - // })) - }) - } - // const dimensions = this.fields.filter(f => f.analyticType === 'dimension'); - // for (let dim of dimensions) { - // this.patterns.push({ - // fields: [dim], - // imp: dim.features.entropy - // }) - // } - this.patterns.sort((a, b) => a.imp - b.imp); - return this.patterns; - } - public createHighOrderPatterns (pattern: IPattern) { - const fieldsInView = pattern.fields; - const viewData = applyFilter(this.dataSource, pattern.filters); - const measures = this.fields.filter(f => f.analyticType === 'measure'); - const patterns: IPattern[] = []; - for (let i = 0; i < measures.length; i++) { - if (!fieldsInView.find(f => f.fid === measures[i].fid)) { - let score = 0; - const T = viewData.map(r => r[measures[i].fid]); - for (let j = 0; j < fieldsInView.length; j++) { - if (fieldsInView[j].analyticType === 'measure') { - const X = viewData.map(r => r[fieldsInView[j].fid]); - score += mic(T, X) - } - } - score /= fieldsInView.length; - patterns.push({ - fields: [...fieldsInView, measures[i]], - filters: pattern.filters, - imp: score - }) - } - } - patterns.sort((a, b) => b.imp - a.imp) - return patterns; - } - public firstPattern () { - const measures = this.fields.filter(f => f.analyticType === 'measure'); - const matrix: number[][] = new Array(measures.length).fill(0).map(() => new Array(measures.length).fill(0)); - for (let i = 0; i < measures.length; i++) { - for (let j = 0 ; j < measures.length; j++) { - const TValues = this.dataSource.map(row => row[measures[i].fid]); - const [T_min, T_max] = getRange(TValues) - const T = binMapShareRange(TValues, T_min, T_max); - const X = this.dataSource.map(row => row[measures[j].fid]); - matrix[i][j] = mic(T, X); - } - } - return matrix; - } - - /** - * 这里其实是比较机制,不是二阶的pattern - * @returns - */ - public secondPattern () { - const { dataSource } = this; - const measures = this.fields.filter(f => f.analyticType === 'measure'); - const patterns: [IFieldMeta, IFieldMeta][] = []; - for (let i = 0; i < measures.length; i++) { - for (let j = i + 1; j < measures.length; j++) { - patterns.push([measures[i], measures[j]]) - } - } - const patternNum = patterns.length; - const normizedScatters = patterns.map(patt => { - return normalizeScatter(dataSource.map(row => [row[patt[0].fid], row[patt[1].fid]])) - }) - - const matrix: number[][] = new Array(patternNum).fill(0).map(() => new Array(patternNum).fill(0)); - for (let i = 0; i < patternNum; i++) { - for (let j = i + 1; j < patternNum; j++) { - matrix[i][j] = matrix[j][i] = l1Dis2(normizedScatters[i], normizedScatters[j]) - } - } - return matrix - } - - public fewatureSelectionForSecondPatternWithSpecifiedViews (patt1: IPattern, patt2: IPattern): { - features: IFieldMeta[]; - score: number; - } | null { - const { dataSource } = this; - const view1Meas = patt1.fields.filter(f => f.analyticType === 'measure'); - const view2Meas = patt2.fields.filter(f => f.analyticType === 'measure'); - const dataView1 = applyFilter(dataSource, patt1.filters); - const dataView2 = applyFilter(dataSource, patt2.filters); - if (patt1.fields.length !== 2 || patt2.fields.length !== 2) throw new Error('View size Not supported yet!') - const dimensions = this.fields.filter(f => f.analyticType === 'dimension'); - let patt1Dist: [number, number][] = dataView1.map(row => view1Meas.map(m => row[m.fid])) as [number, number][]; - let patt2Dist: [number, number][] = dataView2.map(row => view2Meas.map(m => row[m.fid])) as [number, number][]; - - let bestDScore = 0; - let bestDIndex = -1; - for (let k = 0; k < dimensions.length; k++) { - const T1 = dataView1.map(row => row[dimensions[k].fid]) - const T2 = dataView2.map(row => row[dimensions[k].fid]); - const dimScore = incSim(T1, patt1Dist, T2, patt2Dist, dataSource.length) - console.log(dimensions[k].fid, dimScore) - if (dimScore > bestDScore) { - bestDScore = dimScore; - bestDIndex = k; - } - } - if (bestDIndex > -1) { - return { - features: [dimensions[bestDIndex]], - score: bestDScore - } - } else { - return null; - } - } - - public pureFeatureRecommand (pattern: IPattern): IPattern[] { - const { dataSource } = this; - const viewData = applyFilter(dataSource, pattern.filters); - const dimensions = this.fields.filter(f => f.analyticType === 'dimension'); - // const measures = this.fields.filter(f => f.analyticType === 'measure'); - const viewMeasures = pattern.fields.filter(f => f.analyticType === 'measure'); - const viewDimensions = pattern.fields.filter(f => f.analyticType === 'dimension'); - // const nonViewMeasures = measures.filter(f => viewMeasures.findIndex(vf => vf.fid === f.fid) === -1); - const nonViewDimensions = dimensions.filter(f => viewDimensions.findIndex(vf => vf.fid === f.fid) === -1); - const ans: IPattern[] = []; - // fixme: 多个维度时,要渐进的算增益 - nonViewDimensions.forEach(dim => { - const dimValues = viewData.map(row => row[dim.fid]); - let meanScore = 0; - if (viewMeasures.length === 1) { - const mea = viewMeasures[0]; - const meaValues = viewData.map(row => row[mea.fid]); - const score = generalMic(dimValues, meaValues); - meanScore += score; - } else if (viewMeasures.length > 1) { - const meaIds = viewMeasures.map(m => m.fid); - const projections = getCombination(meaIds, 2, 2); - for (let pro of projections) { - const meaProValues: [number, number][] = viewData.map(row => [row[pro[0]], row[pro[1]]]) - const score = generalMatMic(dimValues, meaProValues); - meanScore += score; - } - meanScore /= projections.length - } - ans.push({ - imp: meanScore, - fields: [...pattern.fields, dim], - filters: pattern.filters - }) - }) - // nonViewMeasures.forEach(meaFeat => { - // const dimValues = dataSource.map(row => row[meaFeat.fid]); - // let meanScore = 0; - // viewMeasures.forEach(mea => { - // const meaValues = dataSource.map(row => row[mea.fid]); - // const score = mic(dimValues, meaValues); - // meanScore += score; - // }) - // meanScore /= viewMeasures.length; - // ans.push([[...fields, meaFeat], meanScore]) - // }) - ans.sort((a, b) => b.imp - a.imp) - return ans; - } -} \ No newline at end of file diff --git a/packages/rath-client/src/dev/utils.ts b/packages/rath-client/src/dev/utils.ts deleted file mode 100644 index 7cbb9499..00000000 --- a/packages/rath-client/src/dev/utils.ts +++ /dev/null @@ -1,457 +0,0 @@ -import { Statistics } from 'visual-insights' -import { getRange } from "../utils"; - -const { entropy } = Statistics; - -export function firstWDis (p1: number[], p2: number[]) { - -} - -export function l1Dis(p1: number[], p2: number[]) { - // for (let i ) - let ans = 0; - const safeLen = Math.min(p1.length, p2.length); - for (let i = 0; i < safeLen; i++) { - ans += Math.abs(p1[i] - p2[i]); - } - return ans / 2; -} - -export function l1Dis2(p1: number[][], p2: number[][]) { - let total = 0; - for (let i = 0; i < p1.length; i++ ) { - for (let j = 0; j < p1[i].length; j++) { - total += Math.abs(p1[i][j] - p2[i][j]) - } - } - return total / 2 -} - -export function l2Dis2(p1: number[][], p2: number[][]) { - let total = 0; - for (let i = 0; i < p1.length; i++ ) { - for (let j = 0; j < p1[i].length; j++) { - total += (p1[i][j] - p2[i][j]) ** 2 - } - } - return total / 2 -} - -export function w2dis () { - -} -const BIN_SIZE = 16; -export function bin(nums: number[]): number[] { - const [_min, _max] = getRange(nums) - let step = (_max - _min) / BIN_SIZE; - if (step === 0) return new Array(BIN_SIZE).fill(nums.length / BIN_SIZE) - let dist: number[] = new Array(BIN_SIZE + 1).fill(0); - for (let i = 0; i < nums.length; i++) { - let numIndex = Math.floor((nums[i] - _min) / step) - dist[numIndex % (BIN_SIZE + 1)]++; - } - dist[BIN_SIZE - 1] += dist[BIN_SIZE]; - return dist.slice(0, BIN_SIZE) -} - -export function binShareRange(nums: number[], _min: number, _max: number): number[] { - let step = (_max - _min) / BIN_SIZE; - if (step === 0) return new Array(BIN_SIZE).fill(nums.length / BIN_SIZE) - let dist: number[] = new Array(BIN_SIZE + 1).fill(0); - for (let i = 0; i < nums.length; i++) { - let numIndex = Math.floor((nums[i] - _min) / step) - dist[numIndex % (BIN_SIZE + 1)]++; - } - dist[BIN_SIZE - 1] += dist[BIN_SIZE]; - return dist.slice(0, BIN_SIZE) -} - -export function binMap(nums: number[]): number[] { - const [_min, _max] = getRange(nums); - let step = (_max - _min) / BIN_SIZE; - if (step === 0) return nums.map(n => Math.round(Math.random() * (nums.length - 1))) - let ans: number[] = []; - for (let i = 0; i < nums.length; i++) { - let numIndex = Math.floor((nums[i] - _min) / step) - if (numIndex === BIN_SIZE) { - numIndex = BIN_SIZE - 1; - } - ans.push(numIndex); - } - return ans; -} - -export function binMapShareRange(nums: number[], _min: number, _max: number): number[] { - let step = (_max - _min) / BIN_SIZE; - if (step === 0) return nums.map(n => Math.round(Math.random() * (nums.length - 1))) - let ans: number[] = []; - for (let i = 0; i < nums.length; i++) { - let numIndex = Math.floor((nums[i] - _min) / step) - if (numIndex === BIN_SIZE) { - numIndex = BIN_SIZE - 1; - } - ans.push(numIndex); - } - return ans; -} - -export function rangeNormilize (fl: number[]): number[] { - let _sum = 0; - const pl: number[] = []; - for (let i = 0; i < fl.length; i++) { - _sum += fl[i]; - } - for (let i = 0; i < fl.length; i++) { - pl.push(fl[i] / _sum); - } - return pl -} - -export function mic (T: number[], X: number[]) { - let condH = 0; - const [_min, _max] = getRange(X) - let H = entropy(rangeNormilize(binShareRange(X, _min, _max).filter(v => v > 0))); - for (let i = 0; i < BIN_SIZE; i++) { - const conditionalX = X.filter((x, ti) => T[ti] === i); - const bins = binShareRange(conditionalX, _min, _max).filter(v => v > 0); - const subEnt = entropy(rangeNormilize(bins)) - const px = conditionalX.length / X.length; - condH += px * subEnt; - } - return (H - condH) / Math.log2(BIN_SIZE); -} - -export function generalMic (T: string[], X: number[]) { - let condH = 0; - const [_min, _max] = getRange(X) - let H = entropy(rangeNormilize(binShareRange(X, _min, _max).filter(v => v > 0))); - const uniqueValueSet = new Set(T); - const uniqueValues = [...uniqueValueSet]; - - const dists: Array<{freq: number; bins: number[]}> = []; - - for (let i = 0; i < uniqueValues.length; i++) { - const conditionalX = X.filter((x, ti) => T[ti] === uniqueValues[i]); - const bins = binShareRange(conditionalX, _min, _max) - dists.push({ - freq: conditionalX.length, - bins - }) - } - - dists.sort((a, b) => b.freq - a.freq) - const noise: {freq: number; bins: number[]} = { - freq: 0, - bins: new Array(BIN_SIZE).fill(0) - }; - - for (let i = 0; i < dists.length; i++) { - const { bins, freq } = dists[i] - if (i < BIN_SIZE - 1) { - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } else { - noise.freq += freq - for (let j = 0; j < BIN_SIZE; j++) { - noise.bins[j] += bins[j]; - } - } - } - if (noise.freq > 0) { - const { bins, freq } = noise - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } - - // for (let i = 0; i < uniqueValues.length; i++) { - // const conditionalX = X.filter((x, ti) => T[ti] === uniqueValues[i]); - // const bins = binShareRange(conditionalX, _min, _max).filter(v => v > 0); - // const subEnt = entropy(rangeNormilize(bins)) - // const px = conditionalX.length / X.length; - // condH += px * subEnt; - // } - return (H - condH) / Math.log2(Math.min(BIN_SIZE, uniqueValues.length)); -} - -export function pureGeneralMic (T: string[], X: number[]) { - let condH = 0; - const [_min, _max] = getRange(X) - let H = entropy(rangeNormilize(binShareRange(X, _min, _max).filter(v => v > 0))); - const uniqueValueSet = new Set(T); - const uniqueValues = [...uniqueValueSet]; - - const dists: Array<{freq: number; bins: number[]}> = []; - - for (let i = 0; i < uniqueValues.length; i++) { - const conditionalX = X.filter((x, ti) => T[ti] === uniqueValues[i]); - const bins = binShareRange(conditionalX, _min, _max) - dists.push({ - freq: conditionalX.length, - bins - }) - } - - - for (let i = 0; i < dists.length; i++) { - const { bins, freq } = dists[i] - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } - - // for (let i = 0; i < uniqueValues.length; i++) { - // const conditionalX = X.filter((x, ti) => T[ti] === uniqueValues[i]); - // const bins = binShareRange(conditionalX, _min, _max).filter(v => v > 0); - // const subEnt = entropy(rangeNormilize(bins)) - // const px = conditionalX.length / X.length; - // condH += px * subEnt; - // } - return (H - condH) -} - -export function pureGeneralConditionH (T: string[], X: number[]) { - let condH = 0; - const [_min, _max] = getRange(X) - const uniqueValueSet = new Set(T); - const uniqueValues = [...uniqueValueSet]; - - const dists: Array<{freq: number; bins: number[]}> = []; - - for (let i = 0; i < uniqueValues.length; i++) { - const conditionalX = X.filter((x, ti) => T[ti] === uniqueValues[i]); - const bins = binShareRange(conditionalX, _min, _max) - dists.push({ - freq: conditionalX.length, - bins - }) - } - - - dists.sort((a, b) => b.freq - a.freq) - const noise: {freq: number; bins: number[]} = { - freq: 0, - bins: new Array(BIN_SIZE).fill(0) - }; - - for (let i = 0; i < dists.length; i++) { - const { bins, freq } = dists[i] - if (i < BIN_SIZE - 1) { - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } else { - noise.freq += freq - for (let j = 0; j < BIN_SIZE; j++) { - noise.bins[j] += bins[j]; - } - } - } - if (noise.freq > 0) { - const { bins, freq } = noise - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } - - return condH -} - -export function normalizeScatter (points: [number, number][]) { - let maxX = -Infinity; - let maxY = -Infinity; - let minX = Infinity; - let minY = Infinity; - for (let i = 0; i < points.length; i++) { - maxX = Math.max(points[i][0], maxX) - maxY = Math.max(points[i][1], maxY) - minX = Math.min(points[i][0], minX) - minY = Math.min(points[i][1], minY) - } - const stepX = (maxX - minX) / BIN_SIZE; - const stepY = (maxY - minY) / BIN_SIZE; - const matrix: number[][] = new Array(BIN_SIZE + 1).fill(0).map(() => new Array(BIN_SIZE + 1).fill(0)) - for (let i = 0; i < points.length; i++) { - // matrix[] - const indexX = Math.floor((points[i][0] - minX) / stepX); - const indexY = Math.floor((points[i][1] - minY) / stepY); - matrix[indexX][indexY]++; - } - for (let i = 0; i <= BIN_SIZE; i++) { - matrix[i][BIN_SIZE - 1] += matrix[i][BIN_SIZE] - matrix[BIN_SIZE - 1][i] += matrix[BIN_SIZE][i] - } - let pbMatrix: number[][] = new Array(BIN_SIZE).fill(0).map(() => new Array(BIN_SIZE).fill(0)) - for (let i = 0; i < BIN_SIZE; i++) { - for (let j = 0; j < BIN_SIZE; j++) { - pbMatrix[i][j] = matrix[i][j] / points.length; - } - } - return pbMatrix -} - -export function incSim (TX: string[], pointsX: [number, number][], TY: string[], pointsY: [number, number][], TSize: number) { - const S = l2Dis2(normalizeScatter(pointsX), normalizeScatter(pointsY)); - let groupsX: Map = new Map(); - let groupsY: Map = new Map(); - for (let i = 0; i < TX.length; i++) { - if (!groupsX.has(TX[i])) { - groupsX.set(TX[i], []) - } - groupsX.get(TX[i])!.push(pointsX[i]) - } - for (let i = 0; i < TY.length; i++) { - if (!groupsY.has(TY[i])) { - groupsY.set(TY[i], []) - } - groupsY.get(TY[i])!.push(pointsY[i]) - } - - let condS = 0; - for (let [t, vecX] of groupsX.entries()) { - let p = vecX.length / TSize; - if (p === 0) continue; - if (vecX.length < BIN_SIZE ** 2) { - condS += p; - continue; - } - if (!groupsY.has(t)) { - condS += p; - continue; - } - const vecY = groupsY.get(t)!; - condS += (p * l2Dis2(normalizeScatter(vecX), normalizeScatter(vecY))) - } - for (let [t, vecY] of groupsY.entries()) { - let p = vecY.length / TSize; - if (p === 0) continue; - if (!groupsX.has(t)) { - condS += p; - continue; - } - } - // for (let [, pair] of groups.entries()) { - // let p = pair.X.length / pointsX.length; - // if (p === 0) continue; - // if (pair.X.length < BIN_SIZE ** 2) { - // condS += p; - // continue; - // } - // if () - // // let p = 1 / groups.size - // console.log(pair.X, pair.Y) - // condS += (p * l2Dis2(normalizeScatter(pair.X), normalizeScatter(pair.Y))); - // } - return S - condS; -} - -// type ITensor = Array; - -// function initTensor (order: number, size = BIN_SIZE): ITensor { -// if (order === 1) { -// return new Array(size).fill(0); -// } -// const tensor: ITensor = []; -// for (let i = 0; i < size; i++) { -// tensor.push(initTensor(order - 1, size)); -// } -// return tensor; -// } -export function initRanges (vals: number[][], order: number): [number, number][] { - const ranges: [number, number][] = []; - for (let od = 0; od < order; od++) { - ranges.push(getRange(vals.map(v => v[od]))) - } - return ranges; -} - -// function TensorCellAdd (tensor: ITensor, loc: number[], addVal: number) { - -// } -// function highOrderBinShareRange(vals: number[][], ranges: [number, number][]): number[][] { -// let order = ranges.length; -// const tensor: ITensor = initTensor(order); -// const steps = ranges.map(r => (r[1] - r[0]) / BIN_SIZE); -// for (let i = 0; i < vals.length; i++) { -// for (let j = 0; j < vals[i].length; j++) { -// // tensor -// } -// } -// return [] -// } - -// export function highOrderGeneralMic (T: string[], measures: number[][]) { -// if (measures.length === 0) return 0; -// const measureNumber = measures[0].length; -// const measureBinSize = BIN_SIZE ** measureNumber; -// } -const BIN_SIZE_FOR_MAT = BIN_SIZE / 2; -// const BIN_SIZE_FOR_MAT = BIN_SIZE; -export function matrixBinShareRange (values: [number, number][], ranges: [number, number][]) { - const binMat: number[][] = new Array(BIN_SIZE_FOR_MAT + 1).fill(0).map(() => new Array(BIN_SIZE_FOR_MAT + 1).fill(0)); - const stepX = (ranges[0][1] - ranges[0][0]) / BIN_SIZE_FOR_MAT; - const stepY = (ranges[1][1] - ranges[1][0]) / BIN_SIZE_FOR_MAT; - for (let i = 0; i < values.length; i++) { - const indX = Math.floor((values[i][0] - ranges[0][0]) / stepX); - const indY = Math.floor((values[i][1] - ranges[1][0]) / stepY); - binMat[indY][indX]++; - } - for (let i = 0; i < BIN_SIZE_FOR_MAT + 1; i++) { - binMat[i][BIN_SIZE_FOR_MAT - 1] += binMat[i][BIN_SIZE_FOR_MAT] - } - for (let i = 0; i < BIN_SIZE_FOR_MAT; i++) { - binMat[BIN_SIZE_FOR_MAT - 1][i] += binMat[BIN_SIZE_FOR_MAT][i]; - } - return binMat.slice(0, BIN_SIZE_FOR_MAT).map(row => row.slice(0, BIN_SIZE_FOR_MAT)); -} - -export function generalMatMic (T: string[], X: [number, number][]) { - let condH = 0; - const ranges: [number, number][] = initRanges(X, 2); - - let H = entropy(rangeNormilize(matrixBinShareRange(X, ranges).flatMap(v => v).filter(v => v > 0))); - const uniqueValueSet = new Set(T); - const uniqueValues = [...uniqueValueSet]; - - const dists: Array<{freq: number; bins: number[]}> = []; - - for (let i = 0; i < uniqueValues.length; i++) { - const conditionalX = X.filter((x, ti) => T[ti] === uniqueValues[i]); - // const bins = binShareRange(conditionalX, _min, _max) - const bins = matrixBinShareRange(conditionalX, ranges).flatMap(v => v); - dists.push({ - freq: conditionalX.length, - bins - }) - } - - dists.sort((a, b) => b.freq - a.freq) - const noise: {freq: number; bins: number[]} = { - freq: 0, - bins: new Array(BIN_SIZE_FOR_MAT * BIN_SIZE_FOR_MAT).fill(0) - }; - - for (let i = 0; i < dists.length; i++) { - const { bins, freq } = dists[i] - if (i < BIN_SIZE - 1) { - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } else { - noise.freq += freq - for (let j = 0; j < BIN_SIZE_FOR_MAT * BIN_SIZE_FOR_MAT; j++) { - noise.bins[j] += bins[j]; - } - } - } - if (noise.freq > 0) { - const { bins, freq } = noise - const subEnt = entropy(rangeNormilize(bins.filter(v => v > 0))) - const px = freq / X.length; - condH += px * subEnt; - } - - - return (H - condH) / Math.log2(uniqueValues.length); -} diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx index bc596c0f..130ac685 100644 --- a/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/mainCanvas.tsx @@ -2,7 +2,7 @@ import { observer } from 'mobx-react-lite'; import { Resizable } from 're-resizable'; import React, { useMemo } from 'react'; import ReactVega from '../../../components/react-vega'; -import { IPattern } from '../../../dev'; +import { IPattern } from '@kanaries/loa'; import { IResizeMode, IRow } from '../../../interfaces'; import { distVis } from '../../../queries/distVis'; import { labDistVis } from '../../../queries/labdistVis'; diff --git a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx index 4f1d64ac..a70a58f2 100644 --- a/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx +++ b/packages/rath-client/src/pages/semiAutomation/focusZone/miniFloatCanvas.tsx @@ -5,7 +5,7 @@ import styled from 'styled-components'; import intl from 'react-intl-universal' import ReactVega from '../../../components/react-vega'; -import { IPattern } from '../../../dev'; +import { IPattern } from '@kanaries/loa'; import { IRow } from '../../../interfaces'; import { distVis } from '../../../queries/distVis'; import { labDistVis } from '../../../queries/labdistVis'; diff --git a/packages/rath-client/src/pages/semiAutomation/mainCanvas/index.tsx b/packages/rath-client/src/pages/semiAutomation/mainCanvas/index.tsx index f5a21168..44d80519 100644 --- a/packages/rath-client/src/pages/semiAutomation/mainCanvas/index.tsx +++ b/packages/rath-client/src/pages/semiAutomation/mainCanvas/index.tsx @@ -2,7 +2,7 @@ import { observer } from 'mobx-react-lite'; import { Resizable } from 're-resizable'; import React, { useMemo } from 'react'; import ReactVega from '../../../components/react-vega'; -import { IPattern } from '../../../dev'; +import { IPattern } from '@kanaries/loa'; import { IResizeMode, IRow } from '../../../interfaces'; import { distVis } from '../../../queries/distVis'; import { labDistVis } from '../../../queries/labdistVis'; diff --git a/packages/rath-client/src/pages/semiAutomation/utils.ts b/packages/rath-client/src/pages/semiAutomation/utils.ts index 9c820832..9b3f7056 100644 --- a/packages/rath-client/src/pages/semiAutomation/utils.ts +++ b/packages/rath-client/src/pages/semiAutomation/utils.ts @@ -1,4 +1,4 @@ -import { IFilter } from "../../dev"; +import { IFilter } from "@kanaries/loa"; import { IRow } from "../../interfaces"; // export function applyFilter (dataSource: IRow[], filters?: IFilter[]): IRow[] { diff --git a/packages/rath-client/src/queries/distVis.ts b/packages/rath-client/src/queries/distVis.ts index 08ac7801..a275cc12 100644 --- a/packages/rath-client/src/queries/distVis.ts +++ b/packages/rath-client/src/queries/distVis.ts @@ -1,7 +1,7 @@ /** * distVis 是分布式可视化的推荐,是比较新的模块,目前暂时用于dev模块,即voyager模式下的测试。 */ -import { IPattern } from "../dev"; +import { IPattern } from '@kanaries/loa'; import { IFieldMeta, IResizeMode } from "../interfaces"; import { encodingDecorate } from "./base/utils"; import { applyInteractiveParams2DistViz, applySizeConfig2DistViz } from "./distribution/utils"; diff --git a/packages/rath-client/src/queries/labdistVis.ts b/packages/rath-client/src/queries/labdistVis.ts index c54acc9c..958abe57 100644 --- a/packages/rath-client/src/queries/labdistVis.ts +++ b/packages/rath-client/src/queries/labdistVis.ts @@ -2,8 +2,8 @@ * distVis 是分布式可视化的推荐,是比较新的模块,目前暂时用于dev模块,即voyager模式下的测试。 */ import { Statistics } from 'visual-insights' -import { IPattern } from "../dev"; -import { bin, binMap, mic, pureGeneralMic, rangeNormilize } from "../dev/utils"; +import { IPattern } from '@kanaries/loa'; +import { bin, binMap, mic, pureGeneralMic, rangeNormilize } from '@kanaries/loa'; import { IFieldMeta, IResizeMode, IRow } from "../interfaces"; import { deepcopy } from "../utils"; import { encodingDecorate } from "./base/utils"; diff --git a/packages/rath-client/src/store/discovery/localTypes.ts b/packages/rath-client/src/store/discovery/localTypes.ts index 3ea51a3e..654d0d8b 100644 --- a/packages/rath-client/src/store/discovery/localTypes.ts +++ b/packages/rath-client/src/store/discovery/localTypes.ts @@ -1,4 +1,4 @@ -import { IPattern } from "../../dev"; +import { IPattern } from '@kanaries/loa'; import { IResizeMode } from "../../interfaces"; export interface ISetting { diff --git a/packages/rath-client/src/store/discovery/mainStore.ts b/packages/rath-client/src/store/discovery/mainStore.ts index bcbc89cd..2d726ecf 100644 --- a/packages/rath-client/src/store/discovery/mainStore.ts +++ b/packages/rath-client/src/store/discovery/mainStore.ts @@ -1,6 +1,6 @@ import produce from "immer"; import { makeAutoObservable, observable, reaction, runInAction } from "mobx"; -import { IPattern } from "../../dev"; +import { IPattern } from "@kanaries/loa"; import { IFieldMeta, IResizeMode } from "../../interfaces"; import { distVis } from "../../queries/distVis"; import { labDistVis } from "../../queries/labdistVis"; diff --git a/packages/rath-client/src/workers/fieldsSummary.worker.js b/packages/rath-client/src/workers/fieldsSummary.worker.js index f843f4aa..4e7645ff 100644 --- a/packages/rath-client/src/workers/fieldsSummary.worker.js +++ b/packages/rath-client/src/workers/fieldsSummary.worker.js @@ -1,7 +1,7 @@ /* eslint no-restricted-globals: 0 */ import { UnivariateSummary } from 'visual-insights'; import { Statistics } from 'visual-insights'; -import { bin, rangeNormilize } from '../dev/utils'; +import { bin, rangeNormilize } from '@kanaries/loa'; import { timer } from './timer'; const { getAllFieldsDistribution, getAllFieldTypes, getAllFieldsEntropy } = UnivariateSummary; diff --git a/packages/rath-client/src/workers/footman/service.ts b/packages/rath-client/src/workers/footman/service.ts index fda7bcc0..78e729fb 100644 --- a/packages/rath-client/src/workers/footman/service.ts +++ b/packages/rath-client/src/workers/footman/service.ts @@ -1,4 +1,4 @@ -import { IPattern, NextVICore } from "../../dev"; +import { NextVICore, IPattern } from '@kanaries/loa' import { IFieldMeta, IRow } from "../../interfaces"; export interface IFootmanProps { diff --git a/yarn.lock b/yarn.lock index b3dd988c..326ec978 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2244,6 +2244,11 @@ dependencies: localforage "^1.10.0" +"@kanaries/loa@^0.0.2": + version "0.0.2" + resolved "https://registry.npmmirror.com/@kanaries/loa/-/loa-0.0.2.tgz#7f288fe6dc978fb09ced66849093e2ccfe2ae7bd" + integrity sha512-7Q5UMcA1fVyrjSm6fTSesw/fXTt3PEbxHhoWYsMrE64rVOhCoLxxRcJ8+xLAH7wVwUVRQN2VNokb3E5Jck9X5Q== + "@kanaries/web-data-loader@0.1.5": version "0.1.5" resolved "https://registry.npmmirror.com/@kanaries/web-data-loader/-/web-data-loader-0.1.5.tgz#10d8d677492919b0838b9429d28cbf2b7d726100"