diff --git a/web/src/__tests__/components/Lineage.test.tsx b/web/src/__tests__/components/Lineage.test.tsx index 57a8f62589..6de3231e5d 100644 --- a/web/src/__tests__/components/Lineage.test.tsx +++ b/web/src/__tests__/components/Lineage.test.tsx @@ -1,97 +1,64 @@ // Copyright 2018-2023 contributors to the Marquez project // SPDX-License-Identifier: Apache-2.0 -import React from 'react' -import Lineage, { LineageProps, getSelectedPaths, initGraph, buildGraphAll } from '../../components/lineage/Lineage' -import { LineageNode } from '../../components/lineage/types' -import { render } from '@testing-library/react' -import { createBrowserHistory } from 'history' -import createSagaMiddleware from 'redux-saga' -import { createRouterMiddleware } from '@lagunovsky/redux-react-router' -import createRootReducer from '../../store/reducers' -import { composeWithDevTools } from '@redux-devtools/extension' -import { applyMiddleware, createStore } from 'redux' -import { Provider } from 'react-redux' -import { MqNode } from '../../components/lineage/types' +import { getSelectedPaths, initGraph, buildGraphAll } from '../../components/lineage/Lineage' +import { LineageNode, MqNode } from '../../components/lineage/types' import { graphlib } from 'dagre' -import rootSaga from '../../store/sagas' -const mockGraphWithCycle = [ - { - id: 'job_foo', - inEdges: [ - { - origin: 'dataset_foo', - destination: 'job_foo' - } - ], - outEdges: [ - { - origin: 'job_foo', - destination: 'dataset_bar' - } - ] - }, - { - id: 'dataset_bar', - inEdges: [ - { - origin: 'job_foo', - destination: 'dataset_bar' - } - ], - outEdges: [ - { - origin: 'dataset_bar', - destination: 'job_bar' - } - ] - }, - { - id: 'job_bar', - inEdges: [ - { - origin: 'dataset_bar', - destination: 'job_bar' - } - ], - outEdges: [ - { - origin: 'job_bar', - destination: 'dataset_foo' - } - ] - }, - { - id: 'dataset_foo', - inEdges: [ - { - origin: 'job_bar', - destination: 'dataset_foo' - } - ], - outEdges: [ - { - origin: 'dataset_foo', - destination: 'job_foo' - } - ] +class MockEdge { + origin: string + destination: string + + constructor(origin, destination) { + this.origin = origin + this.destination = destination } -] +} + +class MockNode implements Partial { + id: string + inEdges: MockEdge[] + outEdges: MockEdge[] + + constructor(id, prev: string[], next: string[]) { + this.id = id + this.inEdges = prev ? prev.map(p => new MockEdge(p, id)) : ([] as MockEdge[]) + this.outEdges = next ? next.map(n => new MockEdge(id, n)) : ([] as MockEdge[]) + } +} + +const mockGraphWithCycle = [ + new MockNode('1', ['3'], ['2']), + new MockNode('2', ['1'], ['3']), + new MockNode('3', ['2'], ['1']) +] as LineageNode[] + +const mockGraphWithoutCycle = [ + new MockNode('1', [], ['2', '4']), + new MockNode('2', ['1'], ['3']), + new MockNode('3', ['2'], []), + new MockNode('4', ['1'], []) +] as LineageNode[] describe('Lineage Component', () => { - const selectedNode = 'job_foo' - let g: graphlib.Graph + const selectedNode = '1' + let graphWithCycle: graphlib.Graph beforeEach(() => { - g = initGraph() - buildGraphAll(g, mockGraphWithCycle, (gResult: graphlib.Graph) => { - g = gResult - }) + graphWithCycle = initGraph() + buildGraphAll( + graphWithCycle, + mockGraphWithCycle, + true, + selectedNode, + (gResult: graphlib.Graph) => { + graphWithCycle = gResult + } + ) }) it("doesn't follow cycles in the lineage graph", () => { - const paths = getSelectedPaths(g, selectedNode) + const paths = getSelectedPaths(graphWithCycle, selectedNode) const pathCounts = paths.reduce((acc, p) => { const pathId = p.join(':') @@ -103,19 +70,32 @@ describe('Lineage Component', () => { }) it('renders a valid cycle', () => { - const actualPaths = getSelectedPaths(g, selectedNode) + const actualPaths = getSelectedPaths(graphWithCycle, selectedNode) const expectedPaths = [ - ['job_foo', 'dataset_bar'], - ['dataset_bar', 'job_bar'], - ['job_bar', 'dataset_foo'], - ['dataset_foo', 'job_foo'], - ['dataset_foo', 'job_foo'], - ['job_bar', 'dataset_foo'], - ['dataset_bar', 'job_bar'], - ['job_foo', 'dataset_bar'] + ['1', '2'], + ['2', '3'], + ['3', '1'], + ['3', '1'], + ['2', '3'], + ['1', '2'] ] expect(actualPaths).toEqual(expectedPaths) }) + + it('includes nodes in selected path when fullGraph is true', () => { + const g = initGraph() + buildGraphAll(g, mockGraphWithoutCycle, true, '3', () => null) + + expect(g.node('4')).toBeDefined() + }) + + it('exclude nodes not in selected path when fullGraph is false', () => { + const g = initGraph() + + buildGraphAll(g, mockGraphWithoutCycle, false, '3', () => null) + + expect(g.node('4')).toBeUndefined() + }) }) diff --git a/web/src/components/full-graph-switch/FullGraphSwitch.tsx b/web/src/components/full-graph-switch/FullGraphSwitch.tsx new file mode 100644 index 0000000000..00f987b2c2 --- /dev/null +++ b/web/src/components/full-graph-switch/FullGraphSwitch.tsx @@ -0,0 +1,59 @@ +// Copyright 2018-2023 contributors to the Marquez project +// SPDX-License-Identifier: Apache-2.0 + +import * as Redux from 'redux' +import { Box, FormControlLabel, Switch } from '@mui/material' +import { IState } from '../../store/reducers' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' +import { setShowFullGraph } from '../../store/actionCreators' +import React from 'react' + +interface FullGraphSwitch { + showFullGraph: boolean + setShowFullGraph: (showFullGraph: boolean) => void +} + +const FullGraphSwitch: React.FC = ({ setShowFullGraph, showFullGraph }) => { + const i18next = require('i18next') + const FULL_GRAPH_LABEL = i18next.t('lineage.full_graph_label') + return ( + ({ + display: 'flex', + justifyContent: 'space-evenly', + alignItems: 'center', + zIndex: theme.zIndex.appBar + })} + > + setShowFullGraph(checked)} + color='primary' + /> + } + label={FULL_GRAPH_LABEL} + /> + + ) +} + +const mapStateToProps = (state: IState) => ({ + showFullGraph: state.lineage.showFullGraph +}) + +const mapDispatchToProps = (dispatch: Redux.Dispatch) => + bindActionCreators( + { + setShowFullGraph: setShowFullGraph + }, + dispatch + ) + +export default connect(mapStateToProps, mapDispatchToProps)(FullGraphSwitch) diff --git a/web/src/components/lineage/Lineage.tsx b/web/src/components/lineage/Lineage.tsx index 652d4c25d8..de5733c0ec 100644 --- a/web/src/components/lineage/Lineage.tsx +++ b/web/src/components/lineage/Lineage.tsx @@ -19,13 +19,14 @@ import { fetchLineage, resetLineage, setLineageGraphDepth, - setSelectedNode, + setSelectedNode } from '../../store/actionCreators' import { generateNodeId } from '../../helpers/nodes' import { localPoint } from '@visx/event' import { useParams } from 'react-router-dom' import DepthConfig from './components/depth-config/DepthConfig' import Edge from './components/edge/Edge' +import FullGraphSwitch from '../full-graph-switch/FullGraphSwitch' import MqEmpty from '../core/empty/MqEmpty' import MqText from '../core/text/MqText' import Node from './components/node/Node' @@ -40,6 +41,7 @@ interface StateProps { lineage: LineageGraph selectedNode: string depth: number + showFullGraph: boolean } interface LineageState { @@ -73,6 +75,8 @@ export function initGraph() { export function buildGraphAll( g: graphlib.Graph, graph: LineageNode[], + fullGraph: boolean, + selectedNode: string, callBack: (g: graphlib.Graph) => void ) { // nodes @@ -81,7 +85,7 @@ export function buildGraphAll( label: graph[i].id, data: graph[i].data, width: NODE_SIZE, - height: NODE_SIZE, + height: NODE_SIZE }) } @@ -91,6 +95,10 @@ export function buildGraphAll( g.setEdge(graph[i].inEdges[j].origin, graph[i].id) } } + + if (!fullGraph) { + removeUnselectedNodes(g, selectedNode) + } layout(g) callBack(g) @@ -102,7 +110,7 @@ export function getSelectedPaths(g: graphlib.Graph, selectedNode: string // Sets used to detect cycles and break out of the recursive loop const visitedNodes = { successors: new Set(), - predecessors: new Set(), + predecessors: new Set() } const getSuccessors = (node: string) => { @@ -111,10 +119,10 @@ export function getSelectedPaths(g: graphlib.Graph, selectedNode: string const successors = g?.successors(node) if (successors?.length) { - for (let i = 0; i < node.length - 1; i++) { + for (let i = 0; i < successors.length; i++) { if (successors[i]) { - paths.push([node, successors[i] as unknown as string]) - getSuccessors(successors[i] as unknown as string) + paths.push([node, (successors[i] as unknown) as string]) + getSuccessors((successors[i] as unknown) as string) } } } @@ -126,10 +134,10 @@ export function getSelectedPaths(g: graphlib.Graph, selectedNode: string const predecessors = g?.predecessors(node) if (predecessors?.length) { - for (let i = 0; i < node.length - 1; i++) { + for (let i = 0; i < predecessors.length; i++) { if (predecessors[i]) { - paths.push([predecessors[i] as unknown as string, node]) - getPredecessors(predecessors[i] as unknown as string) + paths.push([(predecessors[i] as unknown) as string, node]) + getPredecessors((predecessors[i] as unknown) as string) } } } @@ -141,6 +149,15 @@ export function getSelectedPaths(g: graphlib.Graph, selectedNode: string return paths } +export function removeUnselectedNodes(g: graphlib.Graph, selectedNode: string) { + const nodesInSelectedPath = new Set(getSelectedPaths(g, selectedNode).flat()) + const nodesToRemove = g.nodes().filter(n => !nodesInSelectedPath.has(n)) + + for (const node of nodesToRemove) { + g.removeNode(node) + } +} + export interface LineageProps extends StateProps, DispatchProps {} let g: graphlib.Graph @@ -149,7 +166,7 @@ const Lineage: React.FC = (props: LineageProps) => { const [state, setState] = React.useState({ graph: g, edges: [], - nodes: [], + nodes: [] }) const { nodeName, namespace, nodeType } = useParams() const mounted = React.useRef(false) @@ -157,6 +174,7 @@ const Lineage: React.FC = (props: LineageProps) => { const prevLineage = React.useRef() const prevDepth = React.useRef() const prevSelectedNode = React.useRef() + const prevShowFullGraph = React.useRef() React.useEffect(() => { if (!mounted.current) { @@ -172,17 +190,24 @@ const Lineage: React.FC = (props: LineageProps) => { // on update if ( (JSON.stringify(props.lineage) !== JSON.stringify(prevLineage.current) || - props.depth !== prevDepth.current) && + props.depth !== prevDepth.current || + props.showFullGraph !== prevShowFullGraph.current) && props.selectedNode ) { g = initGraph() - buildGraphAll(g, props.lineage.graph, (gResult: graphlib.Graph) => { - setState({ - graph: gResult, - edges: getEdges(), - nodes: gResult.nodes().map((v) => gResult.node(v)), - }) - }) + buildGraphAll( + g, + props.lineage.graph, + props.showFullGraph, + props.selectedNode, + (gResult: graphlib.Graph) => { + setState({ + graph: gResult, + edges: getEdges(), + nodes: gResult.nodes().map(v => gResult.node(v)) + }) + } + ) } if (props.selectedNode !== prevSelectedNode.current || props.depth !== prevDepth.current) { props.fetchLineage( @@ -194,9 +219,27 @@ const Lineage: React.FC = (props: LineageProps) => { getEdges() } + if (props.selectedNode !== prevSelectedNode.current && !props.showFullGraph) { + // Always render the graph if the selected node changes and we aren't showing the full graph + // since new nodes may need to be added or removed from view + buildGraphAll( + g, + props.lineage.graph, + props.showFullGraph, + props.selectedNode, + (gResult: graphlib.Graph) => { + setState({ + graph: gResult, + edges: getEdges(), + nodes: gResult.nodes().map(v => gResult.node(v)) + }) + } + ) + } prevLineage.current = props.lineage prevDepth.current = props.depth prevSelectedNode.current = props.selectedNode + prevShowFullGraph.current = props.showFullGraph } }) @@ -210,7 +253,7 @@ const Lineage: React.FC = (props: LineageProps) => { const getEdges = () => { const selectedPaths = getSelectedPaths(g, props.selectedNode) - return g?.edges().map((e) => { + return g?.edges().map(e => { const isSelected = selectedPaths.some((r: any) => e.v === r[0] && e.w === r[1]) return Object.assign(g.edge(e), { isSelected: isSelected }) }) @@ -222,7 +265,7 @@ const Lineage: React.FC = (props: LineageProps) => { {props.selectedNode === null && ( @@ -232,10 +275,20 @@ const Lineage: React.FC = (props: LineageProps) => { )} - + ({ + zIndex: theme.zIndex.appBar + 1, + position: 'absolute', + right: 0, + margin: '1rem 3rem' + })} + > + + + {state?.graph && ( - {(parent) => ( + {parent => ( = (props: LineageProps) => { scaleYMax={MAX_ZOOM} initialTransformMatrix={INITIAL_TRANSFORM} > - {(zoom) => ( + {zoom => (
} > @@ -267,7 +320,7 @@ const Lineage: React.FC = (props: LineageProps) => { onTouchStart={zoom.dragStart} onTouchMove={zoom.dragMove} onTouchEnd={zoom.dragEnd} - onMouseDown={(event) => { + onMouseDown={event => { zoom.dragStart(event) }} onMouseMove={zoom.dragMove} @@ -275,21 +328,21 @@ const Lineage: React.FC = (props: LineageProps) => { onMouseLeave={() => { if (zoom.isDragging) zoom.dragEnd() }} - onDoubleClick={(event) => { + onDoubleClick={event => { const point = localPoint(event) || { x: 0, - y: 0, + y: 0 } zoom.scale({ scaleX: DOUBLE_CLICK_MAGNIFICATION, scaleY: DOUBLE_CLICK_MAGNIFICATION, - point, + point }) }} /> {/* foreground */} - {state?.nodes.map((node) => ( + {state?.nodes.map(node => ( ))} @@ -308,6 +361,7 @@ const mapStateToProps = (state: IState) => ({ lineage: state.lineage.lineage, selectedNode: state.lineage.selectedNode, depth: state.lineage.depth, + showFullGraph: state.lineage.showFullGraph }) const mapDispatchToProps = (dispatch: Redux.Dispatch) => @@ -316,7 +370,7 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) => setSelectedNode: setSelectedNode, fetchLineage: fetchLineage, resetLineage: resetLineage, - setDepth: setLineageGraphDepth, + setDepth: setLineageGraphDepth }, dispatch ) diff --git a/web/src/components/lineage/components/depth-config/DepthConfig.tsx b/web/src/components/lineage/components/depth-config/DepthConfig.tsx index a676fbaaef..cb15fa78e6 100644 --- a/web/src/components/lineage/components/depth-config/DepthConfig.tsx +++ b/web/src/components/lineage/components/depth-config/DepthConfig.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import * as Redux from 'redux' -import { Box, TextField, Typography, createTheme } from '@mui/material' +import { Box, FormControlLabel, TextField, createTheme } from '@mui/material' import { bindActionCreators } from 'redux' import { connect } from 'react-redux' import { setLineageGraphDepth } from '../../../../store/actionCreators' @@ -22,31 +22,38 @@ const DepthConfig: React.FC = ({ setDepth, depth }) => { return ( - {GRAPH_TITLE} - setDepth(isNaN(parseInt(e.target.value)) ? 0 : parseInt(e.target.value))} - variant='outlined' - size='small' - aria-label={GRAPH_TITLE} + setDepth(isNaN(parseInt(e.target.value)) ? 0 : parseInt(e.target.value))} + variant='outlined' + size='small' + aria-label={GRAPH_TITLE} + sx={{ + textAlign: 'center' + }} + inputProps={{ + min: 0, + max: 100 + }} + /> + } + label={GRAPH_TITLE} /> ) @@ -55,7 +62,7 @@ const DepthConfig: React.FC = ({ setDepth, depth }) => { const mapDispatchToProps = (dispatch: Redux.Dispatch) => bindActionCreators( { - setDepth: setLineageGraphDepth, + setDepth: setLineageGraphDepth }, dispatch ) diff --git a/web/src/i18n/config.ts b/web/src/i18n/config.ts index cc3ce26b0e..a2f7fd7f87 100644 --- a/web/src/i18n/config.ts +++ b/web/src/i18n/config.ts @@ -57,7 +57,8 @@ i18next lineage: { empty_title: 'No node selected', empty_body: 'Try selecting a node through search or the jobs or datasets page.', - graph_depth_title: 'Graph Depth' + graph_depth_title: 'Depth', + full_graph_label: 'Full' }, sidenav: { jobs: 'JOBS', diff --git a/web/src/store/actionCreators/actionTypes.ts b/web/src/store/actionCreators/actionTypes.ts index db5fbb98bf..43d78959ff 100644 --- a/web/src/store/actionCreators/actionTypes.ts +++ b/web/src/store/actionCreators/actionTypes.ts @@ -46,6 +46,7 @@ export const FETCH_LINEAGE = 'FETCH_LINEAGE' export const FETCH_LINEAGE_SUCCESS = 'FETCH_LINEAGE_SUCCESS' export const RESET_LINEAGE = 'RESET_LINEAGE' export const SET_LINEAGE_GRAPH_DEPTH = 'SET_LINEAGE_GRAPH_DEPTH' +export const SET_SHOW_FULL_GRAPH = 'SET_SHOW_FULL_GRAPH' // search export const FETCH_SEARCH = 'FETCH_SEARCH' diff --git a/web/src/store/actionCreators/index.ts b/web/src/store/actionCreators/index.ts index 32e98b9541..0220a81703 100644 --- a/web/src/store/actionCreators/index.ts +++ b/web/src/store/actionCreators/index.ts @@ -246,6 +246,11 @@ export const setLineageGraphDepth = (depth: number) => ({ payload: depth }) +export const setShowFullGraph = (showFullGraph: boolean) => ({ + type: actionTypes.SET_SHOW_FULL_GRAPH, + payload: showFullGraph +}) + export const selectNamespace = (namespace: string) => ({ type: actionTypes.SELECT_NAMESPACE, payload: namespace diff --git a/web/src/store/reducers/lineage.ts b/web/src/store/reducers/lineage.ts index 69315a9e58..0d7b13afb7 100644 --- a/web/src/store/reducers/lineage.ts +++ b/web/src/store/reducers/lineage.ts @@ -6,7 +6,8 @@ import { RESET_LINEAGE, SET_BOTTOM_BAR_HEIGHT, SET_LINEAGE_GRAPH_DEPTH, - SET_SELECTED_NODE + SET_SELECTED_NODE, + SET_SHOW_FULL_GRAPH } from '../actionCreators/actionTypes' import { HEADER_HEIGHT } from '../../helpers/theme' import { LineageGraph } from '../../types/api' @@ -18,13 +19,15 @@ export interface ILineageState { selectedNode: Nullable bottomBarHeight: number depth: number + showFullGraph: boolean } const initialState: ILineageState = { lineage: { graph: [] }, selectedNode: null, bottomBarHeight: (window.innerHeight - HEADER_HEIGHT) / 3, - depth: 5 + depth: 5, + showFullGraph: true } type ILineageActions = ReturnType & @@ -52,6 +55,11 @@ export default (state = initialState, action: ILineageActions) => { ...state, depth: action.payload } + case SET_SHOW_FULL_GRAPH: + return { + ...state, + showFullGraph: action.payload + } case RESET_LINEAGE: { return { ...state, lineage: { graph: [] } } }