Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: supporting multi dataset #351

Closed
wants to merge 33 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
04855b3
feat: add iso datetime function
islxyqwe May 20, 2024
336ceb2
fix: dataTable
islxyqwe May 21, 2024
4c94117
fix: color combo box
islxyqwe May 21, 2024
d8b0096
feat: add max width for table cells
islxyqwe May 21, 2024
5f2e2d7
chore: imporove dataTable when embed
islxyqwe May 22, 2024
8650a4d
Merge branch 'feat-table-max-width'
islxyqwe May 22, 2024
57c9d16
chore: remove read * from workflow
islxyqwe Mar 13, 2024
0a63f3d
feat: multi dataset join
islxyqwe Mar 25, 2024
f3132a2
chore: add computed field dataset
islxyqwe Mar 25, 2024
1e4d300
fix: filter tab
islxyqwe Mar 27, 2024
6cfa2a7
fix: fold bugs
islxyqwe Mar 27, 2024
35e4f6a
fix: fold types
islxyqwe Mar 27, 2024
dbcd1ca
fix: merge errors
islxyqwe Apr 1, 2024
ae353ca
fix: datasource segment
islxyqwe Apr 1, 2024
c1a4404
chore: prefix to suffix in query
islxyqwe Apr 1, 2024
5a5a171
save of datasource segment
islxyqwe Apr 12, 2024
8dc872b
fix: preview datasets
islxyqwe Apr 12, 2024
b4ff8ad
fix: foreign dialog
islxyqwe Apr 12, 2024
eb76435
fix: update dsl parser
islxyqwe Apr 18, 2024
aa4672c
fix: show dataset name only at multi dataset mode
islxyqwe Apr 18, 2024
a459054
fix: chat & pure renderer with multi info
islxyqwe Apr 19, 2024
a89f59a
fix: table walker
islxyqwe Apr 19, 2024
99c05e0
fix: merge errors
islxyqwe May 22, 2024
8be9565
Merge branch 'feat-iso-datetime' into multi-dataset
islxyqwe May 22, 2024
c76152c
chore: change multi dataset fields base select
islxyqwe May 22, 2024
06e899e
fix: time unit
islxyqwe May 22, 2024
e496e02
chore: change field list
islxyqwe May 22, 2024
1f6b09b
fix: data board
islxyqwe May 22, 2024
d5493d8
chore: change week def to duckdb
islxyqwe May 22, 2024
607c673
fix: pivot table pureRenderer
islxyqwe May 31, 2024
47abbf6
fix: computation field
islxyqwe May 31, 2024
4f141f6
fix: show table summary is not reactive in pivot table (#388)
islxyqwe May 26, 2024
689b28a
chore: disable collapse for pure renderer
islxyqwe May 31, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: multi dataset join
  • Loading branch information
islxyqwe committed May 22, 2024
commit 0a63f3d6be7e8891b7b9d5f4fb7b7b341197fd4c
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -254,7 +254,6 @@ export interface IGWProps {
i18nLang?: string;
i18nResources?: { [lang: string]: Record<string, string | any> };
keepAlive?: boolean | string;
fieldKeyGuard?: boolean;
vizThemeConfig?: IThemeKey;
apperence?: IDarkMode;
storeRef?: React.MutableRefObject<IGlobalStore | null>;
30 changes: 11 additions & 19 deletions packages/graphic-walker/src/App.tsx
Original file line number Diff line number Diff line change
@@ -20,7 +20,6 @@ import GeoConfigPanel from './components/leafletRenderer/geoConfigPanel';
import AskViz from './components/askViz';
import { renderSpec } from './store/visualSpecStore';
import FieldsContextWrapper from './fields/fieldsContext';
import { guardDataKeys } from './utils/dataPrep';
import { getComputation } from './computation/clientComputation';
import LogPanel from './fields/datasetFields/logPanel';
import BinPanel from './fields/datasetFields/binPanel';
@@ -41,7 +40,9 @@ import { VizAppContext } from './store/context';
import { Tabs, TabsList, TabsTrigger } from './components/ui/tabs';
import { ChartPieIcon, CircleStackIcon, ChatBubbleLeftRightIcon } from '@heroicons/react/24/outline';
import { TabsContent } from '@radix-ui/react-tabs';
import MultiDatasetFields from './fields/datasetFields/multi';
import { VegaliteChat } from './components/chat';
import { LinkDataset } from './components/linkDataset';

export type BaseVizProps = IAppI18nProps &
IVizProps &
@@ -68,6 +69,7 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
chart,
vlSpec,
onError,
datasetNames,
} = props;

const { t, i18n } = useTranslation();
@@ -145,12 +147,13 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {

return (
<ErrorContext value={{ reportError }}>
<ErrorBoundary fallback={<div>Something went wrong</div>} onError={props.onError}>
<ErrorBoundary fallback={<div>Something went wrong</div>} onError={console.log}>
<VizAppContext
ComputationContext={wrappedComputation}
themeContext={darkMode}
vegaThemeContext={{ vizThemeConfig: props.vizThemeConfig ?? props.themeConfig ?? props.themeKey }}
portalContainerContext={portal}
DatasetNamesContext={props.datasetNames}
>
<div className={classNames(`App font-sans bg-background text-foreground m-0 p-0`, darkMode === 'dark' ? 'dark' : '')}>
<FieldsContextWrapper>
@@ -214,6 +217,7 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
<RenamePanel />
<ComputedFieldDialog />
<Painter themeConfig={themeConfig} themeKey={themeKey} />
<LinkDataset />
{vizStore.showGeoJSONConfigPanel && <GeoConfigPanel geoList={props.geoList} />}
<div className="sm:flex">
<SideResize
@@ -222,7 +226,8 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
className="min-w-[100%] max-w-full sm:min-w-[96px] sm:max-w-[35%] flex-shrink-0"
handlerClassName="hidden sm:block"
>
<DatasetFields />
{!vizStore.isMultiDataset && <DatasetFields />}
{vizStore.isMultiDataset && <MultiDatasetFields />}
</SideResize>
<SideResize
defaultWidth={180}
@@ -284,7 +289,7 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
});

export function VizAppWithContext(props: IVizAppProps & IComputationProps) {
const { computation, onMetaChange, fieldKeyGuard, keepAlive, storeRef, defaultConfig, ...rest } = props;
const { computation, onMetaChange, keepAlive, storeRef, defaultConfig, ...rest } = props;
// @TODO remove deprecated props
const appearance = props.appearance ?? props.dark;
const data = props.data ?? props.dataSource;
@@ -294,20 +299,7 @@ export function VizAppWithContext(props: IVizAppProps & IComputationProps) {
safeMetas,
onMetaChange: safeOnMetaChange,
} = useMemo(() => {
if (data) {
if (props.fieldKeyGuard) {
const { safeData, safeMetas } = guardDataKeys(data, fields);
return {
safeMetas,
computation: getComputation(safeData),
onMetaChange: (safeFID, meta) => {
const index = safeMetas.findIndex((x) => x.fid === safeFID);
if (index >= 0) {
props.onMetaChange?.(fields[index].fid, meta);
}
},
};
}
if (props.dataSource) {
return {
safeMetas: fields,
computation: getComputation(data),
@@ -319,7 +311,7 @@ export function VizAppWithContext(props: IVizAppProps & IComputationProps) {
computation: props.computation,
onMetaChange: props.onMetaChange,
};
}, [fields, data ? data : props.computation, props.fieldKeyGuard, props.onMetaChange]);
}, [fields, data ? data : props.computation, props.onMetaChange]);

const darkMode = useCurrentMediaTheme(appearance);

59 changes: 24 additions & 35 deletions packages/graphic-walker/src/Renderer.tsx
Original file line number Diff line number Diff line change
@@ -18,13 +18,12 @@ import ReactiveRenderer from './renderer/index';
import { ComputationContext, VizStoreWrapper, useCompututaion, useVizStore, withErrorReport, withTimeout } from './store';
import { mergeLocaleRes, setLocaleLanguage } from './locales/i18n';
import { renderSpec } from './store/visualSpecStore';
import { guardDataKeys } from './utils/dataPrep';
import { getComputation } from './computation/clientComputation';
import { ErrorContext } from './utils/reportError';
import { ErrorBoundary } from 'react-error-boundary';
import Errorpanel from './components/errorpanel';
import { useCurrentMediaTheme } from './utils/media';
import { classNames, getFilterMeaAggKey, parseErrorMessage } from './utils';
import { classNames, getFilterMeaAggKey, isSameField, parseErrorMessage } from './utils';
import { VegaliteMapper } from './lib/vl2gw';
import { newChart } from './models/visSpecHistory';
import { SimpleOneOfSelector, SimpleRange, SimpleSearcher, SimpleTemporalRange } from './fields/filterField/simple';
@@ -138,6 +137,7 @@ export const RendererApp = observer(function VizApp(props: BaseVizProps) {
themeContext={darkMode}
vegaThemeContext={{ vizThemeConfig: vizThemeConfig ?? themeConfig ?? themeKey }}
portalContainerContext={portal}
DatasetNamesContext={props.datasetNames}
>
<div className={`${darkMode === 'dark' ? 'dark' : ''} App font-sans bg-background text-foreground m-0 p-0`}>
<div className="flex flex-col space-y-2 bg-background text-foreground">
@@ -170,12 +170,12 @@ const FilterItem = observer(function FilterItem({ filter, onChange }: { filter:

const computation = useCompututaion();

const originalField = filter.enableAgg ? allFields.find((x) => x.fid === filter.fid) : undefined;
const originalField = filter.enableAgg ? allFields.find(isSameField(filter)) : undefined;
const filterAggName = filter?.enableAgg ? filter.aggName : undefined;

const transformedComputation = useMemo((): IComputationFunction => {
if (originalField && viewDimensions.length > 0) {
const preWorkflow = toWorkflow(
const { workflow, datasets } = toWorkflow(
[],
allFields,
viewDimensions,
@@ -185,24 +185,26 @@ const FilterItem = observer(function FilterItem({ filter, onChange }: { filter:
[],
undefined,
timezoneDisplayOffset
).map((x) => {
if (x.type === 'view') {
return {
...x,
query: x.query.map((q) => {
if (q.op === 'aggregate') {
return { ...q, measures: q.measures.map((m) => ({ ...m, asFieldKey: m.field })) };
}
return q;
}),
};
}
return x;
});
);
return (query) =>
computation({
...query,
workflow: preWorkflow.concat(query.workflow.filter((x) => x.type !== 'transform')),
workflow: workflow
.map((x) => {
if (x.type === 'view') {
return {
...x,
query: x.query.map((q) => {
if (q.op === 'aggregate') {
return { ...q, measures: q.measures.map((m) => ({ ...m, asFieldKey: m.field })) };
}
return q;
}),
};
}
return x;
})
.concat(query.workflow.filter((x) => x.type !== 'transform')),
});
} else {
return computation;
@@ -273,7 +275,7 @@ const FilterSection = observer(function FilterSection() {
export function RendererAppWithContext(
props: IVizAppProps & IComputationProps & { overrideSize?: IVisualLayout['size']; containerClassName?: string; containerStyle?: React.CSSProperties }
) {
const { dark, dataSource, computation, onMetaChange, fieldKeyGuard, keepAlive, storeRef, defaultConfig, ...rest } = props;
const { dark, dataSource, computation, onMetaChange, keepAlive, storeRef, defaultConfig, ...rest } = props;
// @TODO remove deprecated props
const appearance = props.appearance ?? props.dark;
const data = props.data ?? props.dataSource;
@@ -284,20 +286,7 @@ export function RendererAppWithContext(
safeMetas,
onMetaChange: safeOnMetaChange,
} = useMemo(() => {
if (data) {
if (props.fieldKeyGuard) {
const { safeData, safeMetas } = guardDataKeys(data, fields);
return {
safeMetas,
computation: getComputation(safeData),
onMetaChange: (safeFID, meta) => {
const index = safeMetas.findIndex((x) => x.fid === safeFID);
if (index >= 0) {
props.onMetaChange?.(fields[index].fid, meta);
}
},
};
}
if (props.dataSource) {
return {
safeMetas: fields,
computation: getComputation(data),
@@ -309,7 +298,7 @@ export function RendererAppWithContext(
computation: props.computation,
onMetaChange: props.onMetaChange,
};
}, [fields, data ? data : props.computation, props.fieldKeyGuard, props.onMetaChange]);
}, [fields, data ? data : props.computation, props.onMetaChange]);

const darkMode = useCurrentMediaTheme(appearance);

53 changes: 28 additions & 25 deletions packages/graphic-walker/src/Table.tsx
Original file line number Diff line number Diff line change
@@ -4,16 +4,17 @@ import { ErrorBoundary } from 'react-error-boundary';
import { useTranslation } from 'react-i18next';
import { IAppI18nProps, IErrorHandlerProps, IComputationContextProps, ITableProps, ITableSpecProps, IComputationProps, IMutField } from './interfaces';
import { mergeLocaleRes, setLocaleLanguage } from './locales/i18n';
import { useVizStore, withErrorReport, withTimeout, ComputationContext, VizStoreWrapper } from './store';
import { parseErrorMessage } from './utils';
import { useVizStore, withErrorReport, withTimeout, VizStoreWrapper } from './store';
import { getFieldIdentifier, parseErrorMessage } from './utils';
import { ErrorContext } from './utils/reportError';
import { guardDataKeys } from './utils/dataPrep';
import { getComputation } from './computation/clientComputation';
import DatasetTable from './components/dataTable';
import { useCurrentMediaTheme } from './utils/media';
import { toJS } from 'mobx';
import Errorpanel from './components/errorpanel';
import { VizAppContext } from './store/context';
import { DEFAULT_DATASET } from './constants';
import DataTable from './components/dataTable';
import { Tabs, TabsList, TabsTrigger } from './components/ui/tabs';

export type BaseTableProps = IAppI18nProps &
IErrorHandlerProps &
@@ -73,6 +74,9 @@ export const TableApp = observer(function VizApp(props: BaseTableProps) {
const metas = toJS(vizStore.meta);
const [portal, setPortal] = useState<HTMLDivElement | null>(null);

const datasets = Array.from(new Set(metas.map((x) => x.dataset ?? DEFAULT_DATASET)));
const [dataset, setDataset] = useState(datasets[0] ?? DEFAULT_DATASET);

return (
<ErrorContext value={{ reportError }}>
<ErrorBoundary fallback={<div>Something went wrong</div>} onError={props.onError}>
@@ -81,13 +85,27 @@ export const TableApp = observer(function VizApp(props: BaseTableProps) {
themeContext={darkMode}
vegaThemeContext={{ vizThemeConfig: vizThemeConfig ?? themeConfig ?? themeKey }}
portalContainerContext={portal}
DatasetNamesContext={props.datasetNames}
>
<div className={`${darkMode === 'dark' ? 'dark' : ''} App font-sans bg-background text-foreground h-full m-0 p-0`}>
<div className="bg-background text-foreground h-full">
<DatasetTable
onMetaChange={vizStore.onMetaChange ? (fid, fIndex, diffMeta) => {
vizStore.updateCurrentDatasetMetas(fid, diffMeta);
} : undefined}
{datasets.length > 1 && (
<Tabs value={dataset} onValueChange={setDataset}>
<TabsList>
{datasets.map((ds) => (
<TabsTrigger value={ds}>{props.datasetNames?.[ds] ?? ds}</TabsTrigger>
))}
</TabsList>
</Tabs>
)}
<DataTable
onMetaChange={
vizStore.onMetaChange
? (fid, fIndex, diffMeta) => {
vizStore.updateCurrentDatasetMetas(getFieldIdentifier(metas[fIndex]), diffMeta);
}
: undefined
}
size={pageSize}
metas={metas}
computation={wrappedComputation}
@@ -106,7 +124,7 @@ export const TableApp = observer(function VizApp(props: BaseTableProps) {
});

export function TableAppWithContext(props: ITableProps & IComputationProps) {
const { dark, dataSource, computation, onMetaChange, fieldKeyGuard, keepAlive, storeRef, defaultConfig, ...rest } = props;
const { dark, dataSource, computation, onMetaChange, keepAlive, storeRef, defaultConfig, ...rest } = props;
// @TODO remove deprecated props
const appearance = props.appearance ?? props.dark;
const data = props.data ?? props.dataSource;
@@ -118,21 +136,6 @@ export function TableAppWithContext(props: ITableProps & IComputationProps) {
onMetaChange: safeOnMetaChange,
} = useMemo(() => {
if (data) {
if (props.fieldKeyGuard) {
const { safeData, safeMetas } = guardDataKeys(data, fields);
return {
safeMetas,
computation: getComputation(safeData),
onMetaChange: onMetaChange
? (safeFID, meta) => {
const index = safeMetas.findIndex((x) => x.fid === safeFID);
if (index >= 0) {
onMetaChange(fields[index].fid, meta);
}
}
: undefined,
};
}
return {
safeMetas: fields,
computation: getComputation(data),
@@ -144,7 +147,7 @@ export function TableAppWithContext(props: ITableProps & IComputationProps) {
computation: props.computation,
onMetaChange,
};
}, [fields, data ? data : props.computation, props.fieldKeyGuard, onMetaChange]);
}, [fields, data ? data : props.computation, onMetaChange]);

const darkMode = useCurrentMediaTheme(appearance);

11 changes: 6 additions & 5 deletions packages/graphic-walker/src/components/computedField/index.tsx
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { observer } from 'mobx-react-lite';
import React, { useState, useRef, useMemo, useEffect } from 'react';
import { useVizStore } from '../../store';
import { isNotEmpty, parseErrorMessage } from '../../utils';
import { getFieldIdentifier, isNotEmpty, parseErrorMessage } from '../../utils';
import { highlightField } from '../highlightField';
import { aggFuncs, reservedKeywords, sqlFunctions } from '../../lib/sql';
import { COUNT_FIELD_ID, MEA_KEY_ID, MEA_VAL_ID, PAINT_FIELD_ID } from '../../constants';
import { COUNT_FIELD_ID, EMPTY_FIELD_ID, MEA_KEY_ID, MEA_VAL_ID, PAINT_FIELD_ID } from '../../constants';
import { unstable_batchedUpdates } from 'react-dom';
import { Dialog, DialogContent } from '../ui/dialog';
import { Input } from '../ui/input';
@@ -23,6 +23,7 @@ const ComputedFieldDialog: React.FC = observer(() => {
const [sql, setSql] = useState<string>('');
const [error, setError] = useState<string>('');
const ref = useRef<HTMLDivElement>(null);
// TODO bind a dataset to computed field.

const SQLField = useMemo(() => {
const fields = vizStore.allFields
@@ -66,14 +67,14 @@ const ComputedFieldDialog: React.FC = observer(() => {
});
ref.current && (ref.current.innerHTML = '');
} else {
const f = vizStore.allFields.find((x) => x.fid === editingComputedFieldFid);
const f = vizStore.allFields.find((x) => getFieldIdentifier(x) === editingComputedFieldFid);
if (!f || !f.computed || f.expression?.op !== 'expr') {
vizStore.setComputedFieldFid('');
vizStore.setComputedFieldFid(EMPTY_FIELD_ID);
return;
}
const sql = f.expression.params.find((x) => x.type === 'sql');
if (!sql) {
vizStore.setComputedFieldFid('');
vizStore.setComputedFieldFid(EMPTY_FIELD_ID);
return;
}
unstable_batchedUpdates(() => {
Loading