Merge main

Signed-off-by: tito12 <vladyslav.sedenko@gmail.com>

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.30.0...HEAD)
 
+### Fixed
+
+* Add missing database indexes [`#2419`](https://github.com/MarquezProject/marquez/pull/2419) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)  
+  *Create missing indexes on reference columns.*
+
+
 ## [0.30.0](https://github.com/MarquezProject/marquez/compare/0.29.0...0.30.0) - 2023-01-31
 
 ### Added

api/build.gradle

@@ -21,7 +21,7 @@ plugins {
 }
 
 ext {
-    jdbi3Version = '3.36.0'
+    jdbi3Version = '3.37.1'
     prometheusVersion = '0.16.0'
     testcontainersVersion = '1.17.6'
     sentryVersion = '6.13.0'

api/src/main/resources/marquez/db/migration/V58__job_fqn_name_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ON jobs_fqn (namespace_name, job_fqn);

api/src/main/resources/marquez/db/migration/V59.1__column_lineage_add_indexes.sql

@@ -0,0 +1,11 @@
+create index column_lineage_output_dataset_version_uuid_index
+    on column_lineage (output_dataset_version_uuid);
+
+create index column_lineage_output_dataset_field_uuid_index
+    on column_lineage (output_dataset_field_uuid);
+
+create index column_lineage_input_dataset_version_uuid_index
+    on column_lineage (input_dataset_version_uuid);
+
+create index column_lineage_input_dataset_field_uuid_index
+    on column_lineage (input_dataset_field_uuid);

api/src/main/resources/marquez/db/migration/V59.2__facet_tables_indexes.sql

@@ -0,0 +1,14 @@
+create index dataset_facets_dataset_uuid_index
+    on dataset_facets (dataset_uuid);
+
+create index dataset_facets_dataset_version_uuid_index
+    on dataset_facets (dataset_version_uuid);
+
+create index dataset_facets_run_uuid_index
+    on dataset_facets (run_uuid);
+
+create index job_facets_job_uuid_index
+    on job_facets (job_uuid);
+
+create index job_facets_run_uuid_index
+    on job_facets (run_uuid);

build.gradle

@@ -21,7 +21,7 @@ buildscript {
     dependencies {
         classpath 'com.adarshr:gradle-test-logger-plugin:3.2.0'
         classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
-        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.14.0'
+        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.15.0'
     }
 }
 
@@ -58,11 +58,11 @@ subprojects {
         dropwizardVersion = '2.1.4'
         jacocoVersion = '0.8.8'
         junit5Version = '5.9.2'
-        lombokVersion = '1.18.24'
-        mockitoVersion = '5.1.0'
-        openlineageVersion = '0.19.2'
+        lombokVersion = '1.18.26'
+        mockitoVersion = '5.1.1'
+        openlineageVersion = '0.20.6'
         slf4jVersion = '1.7.36'
-        postgresqlVersion = '42.5.1'
+        postgresqlVersion = '42.5.3'
         isReleaseVersion = !version.endsWith('SNAPSHOT')
     }
 

docker/metadata.json

@@ -1358,6 +1358,22 @@
             "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json",
             "name": "food_delivery_db",
             "uri": "postgres://food_delivery:food_delivery@postgres:5432/food_delivery"
+          },
+          "dataQualityAssertions": {
+            "_producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json",
+            "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json",
+            "assertions": [
+              {
+                "assertion": "not_null",
+                "success": false,
+                "column": "driver_id"
+              },
+              {
+                "assertion": "is_string",
+                "success": true,
+                "column": "customer_address"
+              }
+            ]
           }
         }
       }
@@ -1824,4 +1840,4 @@
     },
     "producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json"
   }
-]
+]

renovate.json

@@ -1,5 +1,5 @@
 {
-  "bumpVersion": "patch",
+  "bumpVersion": "minor",
   "extends": [
     "config:base",
     "docker:disable",

web/src/components/core/status/MqStatus.tsx

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { theme } from '../../../helpers/theme'
+import Box from '@material-ui/core/Box'
+import MqText from '../text/MqText'
+import React from 'react'
+import createStyles from '@material-ui/core/styles/createStyles'
+import withStyles, { WithStyles } from '@material-ui/core/styles/withStyles'
+
+const styles = () =>
+  createStyles({
+    type: {
+      display: 'flex',
+      alignItems: 'center',
+      gap: theme.spacing(1)
+    },
+    status: {
+      width: theme.spacing(2),
+      height: theme.spacing(2),
+      borderRadius: '50%'
+    }
+  })
+
+interface OwnProps {
+  color: string | null
+  label?: string
+}
+
+const MqStatus: React.FC<OwnProps & WithStyles<typeof styles>> = ({ label, color, classes }) => {
+  if (!color) {
+    return null
+  }
+  return (
+    <Box className={classes.type}>
+      <Box className={classes.status} style={{ backgroundColor: color }} />
+      {label && <MqText>{label}</MqText>}
+    </Box>
+  )
+}
+
+export default withStyles(styles)(MqStatus)

web/src/components/datasets/DatasetColumnLineage.tsx

@@ -102,7 +102,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(DatasetColumnLineage)
+export default connect(mapStateToProps, mapDispatchToProps)(DatasetColumnLineage)

web/src/components/datasets/DatasetDetailPage.tsx

@@ -15,6 +15,7 @@ import { LineageDataset } from '../lineage/types'
 import { alpha } from '@material-ui/core/styles'
 import { bindActionCreators } from 'redux'
 import { connect } from 'react-redux'
+import { datasetFacetsStatus } from '../../helpers/nodes'
 import {
   deleteDataset,
   dialogToggle,
@@ -31,7 +32,9 @@ import DatasetInfo from './DatasetInfo'
 import DatasetVersions from './DatasetVersions'
 import Dialog from '../Dialog'
 import IconButton from '@material-ui/core/IconButton'
+import MqStatus from '../core/status/MqStatus'
 import MqText from '../core/text/MqText'
+
 import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
 
 const styles = ({ spacing }: ITheme) => {
@@ -143,6 +146,7 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
 
   const firstVersion = versions[0]
   const { name, tags, description } = firstVersion
+  const facetsStatus = datasetFacetsStatus(firstVersion.facets)
 
   return (
     <Box my={2} className={root}>
@@ -202,9 +206,16 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
             </IconButton>
           </Box>
         </Box>
-        <MqText heading font={'mono'}>
-          {name}
-        </MqText>
+        <Box display={'flex'} alignItems={'center'}>
+          {facetsStatus && (
+            <Box mr={1}>
+              <MqStatus color={facetsStatus} />
+            </Box>
+          )}
+          <MqText heading font={'mono'}>
+            {name}
+          </MqText>
+        </Box>
         <Box mb={2}>
           <MqText subdued>{description}</MqText>
         </Box>
@@ -241,7 +252,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(withStyles(styles)(DatasetDetailPage))
+export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(DatasetDetailPage))

web/src/components/datasets/DatasetVersions.tsx

@@ -31,9 +31,8 @@ interface DatasetVersionsProps {
   versions: DatasetVersion[]
 }
 
-const DatasetVersions: FunctionComponent<
-  DatasetVersionsProps & IWithStyles<typeof styles>
-> = props => {
+const DatasetVersions: FunctionComponent<DatasetVersionsProps &
+  IWithStyles<typeof styles>> = props => {
   const { versions, classes } = props
 
   const [infoView, setInfoView] = React.useState<DatasetVersion | null>(null)

web/src/components/jobs/JobDetailPage.tsx

@@ -25,15 +25,16 @@ import {
   resetJobs,
   resetRuns
 } from '../../store/actionCreators'
+import { jobRunsStatus } from '../../helpers/nodes'
 import { theme } from '../../helpers/theme'
 import { useHistory } from 'react-router-dom'
 import CloseIcon from '@material-ui/icons/Close'
 import Dialog from '../Dialog'
 import IconButton from '@material-ui/core/IconButton'
 import MqEmpty from '../core/empty/MqEmpty'
+import MqStatus from '../core/status/MqStatus'
 import MqText from '../core/text/MqText'
 import RunInfo from './RunInfo'
-import RunStatus from './RunStatus'
 import Runs from './Runs'
 
 const styles = ({ spacing }: ITheme) => {
@@ -166,9 +167,9 @@ const JobDetailPage: FunctionComponent<IProps> = props => {
         </Box>
       </Box>
       <Box display={'flex'} alignItems={'center'}>
-        {job.latestRun && (
+        {runs.length && (
           <Box mr={1}>
-            <RunStatus run={job.latestRun} />
+            <MqStatus color={jobRunsStatus(runs)} />
           </Box>
         )}
         <MqText font={'mono'} heading>
@@ -212,7 +213,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
     dispatch
   )
 
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(withStyles(styles)(JobDetailPage))
+export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(JobDetailPage))

web/src/components/jobs/RunStatus.tsx

@@ -3,8 +3,7 @@
 
 import { Box, Theme, Tooltip, WithStyles, createStyles, withStyles } from '@material-ui/core'
 import { Run } from '../../types/api'
-
-import { runColorMap } from '../../helpers/runs'
+import { runStateColor } from '../../helpers/nodes'
 
 import React, { FunctionComponent } from 'react'
 
@@ -26,7 +25,11 @@ const RunStatus: FunctionComponent<RunStatusProps & WithStyles<typeof styles>> =
   const { run, classes } = props
   return (
     <Tooltip title={run.state}>
-      <Box mr={1} className={classes.status} style={{ backgroundColor: runColorMap[run.state] }} />
+      <Box
+        mr={1}
+        className={classes.status}
+        style={{ backgroundColor: runStateColor(run.state) }}
+      />
     </Tooltip>
   )
 }