Merge branch 'main' into feature/java-client-lineage

api/src/main/java/marquez/db/JobDao.java

@@ -283,7 +283,6 @@ INSERT INTO jobs_view AS j (
       namespace_name,
       name,
       description,
-      current_job_context_uuid,
       current_location,
       current_inputs,
       symlink_target_uuid,
@@ -297,7 +296,6 @@ INSERT INTO jobs_view AS j (
       :namespaceName,
       :name,
       :description,
-      null,
       :location,
       :inputs,
       :symlinkTargetId,
@@ -332,7 +330,6 @@ INSERT INTO jobs_view AS j (
       namespace_name,
       name,
       description,
-      current_job_context_uuid,
       current_location,
       current_inputs,
       symlink_target_uuid
@@ -346,7 +343,6 @@ INSERT INTO jobs_view AS j (
       :namespaceName,
       :name,
       :description,
-      null,
       :location,
       :inputs,
       :symlinkTargetId

api/src/main/java/marquez/db/LineageDao.java

@@ -73,10 +73,9 @@ WHERE j.uuid IN (<jobIds>) OR j.symlink_target_uuid IN (<jobIds>)
               WHERE io.job_uuid != l.job_uuid AND
                   array_cat(io.inputs, io.outputs) && array_cat(l.inputs, l.outputs)
                 AND depth < :depth)
-      SELECT DISTINCT ON (j.uuid) j.*, inputs AS input_uuids, outputs AS output_uuids, jc.context
+      SELECT DISTINCT ON (j.uuid) j.*, inputs AS input_uuids, outputs AS output_uuids
       FROM lineage l2
-      INNER JOIN jobs_view j ON j.uuid=l2.job_uuid
-      LEFT JOIN job_contexts jc on jc.uuid = j.current_job_context_uuid;
+      INNER JOIN jobs_view j ON j.uuid=l2.job_uuid;
   """)
   Set<JobData> getLineage(@BindList Set<UUID> jobIds, int depth);
 
@@ -116,11 +115,10 @@ WHERE ds.uuid IN (<dsUuids>)""")
           + "    WHERE j.uuid in (<jobUuid>) OR j.symlink_target_uuid IN (<jobUuid>)\n"
           + "    ORDER BY r.job_name, r.namespace_name, created_at DESC\n"
           + ")\n"
-          + "SELECT r.*, ra.args, ctx.context, f.facets,\n"
+          + "SELECT r.*, ra.args, f.facets,\n"
           + "  r.version AS job_version, ri.input_versions, ro.output_versions\n"
           + "  from latest_runs AS r\n"
           + "LEFT JOIN run_args AS ra ON ra.uuid = r.run_args_uuid\n"
-          + "LEFT JOIN job_contexts AS ctx ON r.job_context_uuid = ctx.uuid\n"
           + "LEFT JOIN LATERAL (\n"
           + "    SELECT im.run_uuid,\n"
           + "           JSON_AGG(json_build_object('namespace', dv.namespace_name,\n"

api/src/main/java/marquez/db/RunDao.java

@@ -207,8 +207,7 @@ LEFT OUTER JOIN (
           + "transitioned_at, "
           + "namespace_name, "
           + "job_name, "
-          + "location, "
-          + "job_context_uuid "
+          + "location "
           + ") VALUES ( "
           + ":runUuid, "
           + ":parentRunUuid, "
@@ -224,8 +223,7 @@ LEFT OUTER JOIN (
           + ":runStateTime, "
           + ":namespaceName, "
           + ":jobName, "
-          + ":location, "
-          + "null "
+          + ":location "
           + ") ON CONFLICT(uuid) DO "
           + "UPDATE SET "
           + "external_id = EXCLUDED.external_id, "
@@ -266,8 +264,7 @@ RunRow upsert(
           + "nominal_end_time, "
           + "namespace_name, "
           + "job_name, "
-          + "location, "
-          + "job_context_uuid "
+          + "location "
           + ") VALUES ( "
           + ":runUuid, "
           + ":parentRunUuid, "
@@ -281,8 +278,7 @@ RunRow upsert(
           + ":nominalEndTime, "
           + ":namespaceName, "
           + ":jobName, "
-          + ":location, "
-          + "null"
+          + ":location "
           + ") ON CONFLICT(uuid) DO "
           + "UPDATE SET "
           + "external_id = EXCLUDED.external_id, "

api/src/main/java/marquez/db/migrations/V44_3_BackfillJobsWithParents.java

@@ -51,10 +51,10 @@ LEFT JOIN LATERAL (
   public static final String INSERT_NEW_JOB_WITH_PARENT =
       """
       INSERT INTO jobs(uuid, type, created_at, updated_at, namespace_uuid, name, description,
-        current_version_uuid, namespace_name, current_job_context_uuid, current_location, current_inputs,
+        current_version_uuid, namespace_name, current_location, current_inputs,
         parent_job_uuid)
       SELECT :uuid, type, created_at, updated_at, namespace_uuid, name, description, current_version_uuid,
-        namespace_name, current_job_context_uuid, current_location, current_inputs, :parent_job_uuid
+        namespace_name, current_location, current_inputs, :parent_job_uuid
         FROM jobs
         WHERE uuid=:job_uuid
         ON CONFLICT (name, namespace_name, parent_job_uuid) DO NOTHING

api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql

@@ -52,7 +52,7 @@ BEGIN
            NEW.description,
            NEW.current_version_uuid,
            NEW.namespace_name,
-           NEW.current_job_context_uuid,
+           NULL,
            NEW.current_location,
            NEW.current_inputs,
            NEW.symlink_target_uuid,
@@ -67,7 +67,6 @@ BEGIN
                             END,
                       type                     = EXCLUDED.type,
                       description              = EXCLUDED.description,
-                      current_job_context_uuid = EXCLUDED.current_job_context_uuid,
                       current_location         = EXCLUDED.current_location,
                       current_inputs           = EXCLUDED.current_inputs,
                       -- update the symlink target if null. otherwise, keep the old value

api/src/main/resources/marquez/db/migration/V64__drop_job_contexts.sql

@@ -0,0 +1 @@
+DROP TABLE job_contexts;

api/src/test/java/marquez/db/BackfillTestUtils.java

@@ -122,8 +122,8 @@ public static UUID writeJob(Jdbi jdbi, String jobName, Instant now, NamespaceRow
         h -> {
           return h.createQuery(
                   """
-                  INSERT INTO jobs (uuid, type, created_at, updated_at, namespace_uuid, name, namespace_name, current_job_context_uuid, current_inputs)
-                  VALUES (:uuid, :type, :now, :now, :namespaceUuid, :name, :namespaceName, null, :currentInputs)
+                  INSERT INTO jobs (uuid, type, created_at, updated_at, namespace_uuid, name, namespace_name, current_inputs)
+                  VALUES (:uuid, :type, :now, :now, :namespaceUuid, :name, :namespaceName, :currentInputs)
                   RETURNING uuid
                   """)
               .bind("uuid", UUID.randomUUID())

web/src/components/core/text/MqText.tsx

@@ -30,6 +30,7 @@ interface OwnProps {
   small?: boolean
   bottomMargin?: boolean
   children: ReactElement | (string | ReactElement)[] | string | string[] | number | undefined | null
+  onClick?: () => void
 }
 
 type MqTextProps = OwnProps
@@ -53,6 +54,7 @@ const MqText: React.FC<MqTextProps> = ({
   highlight,
   color,
   small,
+  onClick,
 }) => {
   const theme = createTheme(useTheme())
 
@@ -143,6 +145,7 @@ const MqText: React.FC<MqTextProps> = ({
   if (heading) {
     return (
       <Typography
+        onClick={onClick}
         variant='h4'
         sx={Object.assign(classesObject.root, classesObject.heading, conditionalClasses)}
         style={style}
@@ -152,7 +155,12 @@ const MqText: React.FC<MqTextProps> = ({
     )
   } else if (link && linkTo) {
     return (
-      <LinkRouter to={linkTo} aria-disabled={disabled} style={{ textDecoration: 'none' }}>
+      <LinkRouter
+        to={linkTo}
+        aria-disabled={disabled}
+        style={{ textDecoration: 'none' }}
+        onClick={onClick}
+      >
         <Box
           component='span'
           sx={Object.assign(classesObject.root, classesObject.link, conditionalClasses)}
@@ -164,6 +172,7 @@ const MqText: React.FC<MqTextProps> = ({
   } else if (link && href) {
     return (
       <Link
+        onClick={onClick}
         href={href}
         target={'_blank'}
         rel='noopener noreferrer'
@@ -174,7 +183,11 @@ const MqText: React.FC<MqTextProps> = ({
     )
   } else {
     return (
-      <Box sx={Object.assign(classesObject.root, conditionalClasses)} style={style}>
+      <Box
+        onClick={onClick}
+        sx={Object.assign(classesObject.root, conditionalClasses)}
+        style={style}
+      >
         {children}
       </Box>
     )

web/src/components/datasets/DatasetDetailPage.tsx

@@ -17,6 +17,7 @@ import {
   fetchDatasetVersions,
   resetDataset,
   resetDatasetVersions,
+  setTabIndex,
 } from '../../store/actionCreators'
 import { useNavigate } from 'react-router-dom'
 import CloseIcon from '@mui/icons-material/Close'
@@ -29,14 +30,16 @@ import MqStatus from '../core/status/MqStatus'
 import MqText from '../core/text/MqText'
 
 import { useTheme } from '@emotion/react'
-import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
+import Io from '../io/Io'
+import React, { ChangeEvent, FunctionComponent, useEffect } from 'react'
 
 interface StateProps {
   lineageDataset: LineageDataset
   versions: DatasetVersion[]
   versionsLoading: boolean
   datasets: IState['datasets']
   display: IState['display']
+  tabIndex: IState['lineage']['tabIndex']
 }
 
 interface DispatchProps {
@@ -45,6 +48,7 @@ interface DispatchProps {
   resetDataset: typeof resetDataset
   deleteDataset: typeof deleteDataset
   dialogToggle: typeof dialogToggle
+  setTabIndex: typeof setTabIndex
 }
 
 type IProps = StateProps & DispatchProps
@@ -68,6 +72,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     versions,
     versionsLoading,
     lineageDataset,
+    tabIndex,
+    setTabIndex,
   } = props
   const navigate = useNavigate()
   const i18next = require('i18next')
@@ -92,9 +98,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     []
   )
 
-  const [tab, setTab] = React.useState(0)
-  const handleChange = (event: ChangeEvent, newValue: SetStateAction<number>) => {
-    setTab(newValue)
+  const handleChange = (_: ChangeEvent, newValue: number) => {
+    setTabIndex(newValue)
   }
 
   if (versionsLoading) {
@@ -149,20 +154,26 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
         )}
         <Box display={'flex'} justifyContent={'space-between'} mb={2}>
           <Box sx={{ borderBottom: 1, borderColor: 'divider' }}>
-            <Tabs value={tab} onChange={handleChange} textColor='primary' indicatorColor='primary'>
+            <Tabs
+              value={tabIndex}
+              onChange={handleChange}
+              textColor='primary'
+              indicatorColor='primary'
+            >
               <Tab
                 label={i18next.t('datasets.latest_tab')}
                 {...a11yProps(0)}
                 disableRipple={true}
               />
+              <Tab label={'I/O'} {...a11yProps(1)} disableRipple={true} />
               <Tab
                 label={i18next.t('datasets.history_tab')}
-                {...a11yProps(1)}
+                {...a11yProps(2)}
                 disableRipple={true}
               />
               <Tab
                 label={i18next.t('datasets.column_lineage_tab')}
-                {...a11yProps(1)}
+                {...a11yProps(3)}
                 disableRipple={true}
               />
             </Tabs>
@@ -214,15 +225,16 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
           <MqText subdued>{description}</MqText>
         </Box>
       </Box>
-      {tab === 0 && (
+      {tabIndex === 0 && (
         <DatasetInfo
           datasetFields={firstVersion.fields}
           facets={firstVersion.facets}
           run={firstVersion.createdByRun}
         />
       )}
-      {tab === 1 && <DatasetVersions versions={props.versions} />}
-      {tab === 2 && <DatasetColumnLineage lineageDataset={props.lineageDataset} />}
+      {tabIndex === 1 && <Io />}
+      {tabIndex === 2 && <DatasetVersions versions={props.versions} />}
+      {tabIndex === 3 && <DatasetColumnLineage lineageDataset={props.lineageDataset} />}
     </Box>
   )
 }
@@ -232,6 +244,7 @@ const mapStateToProps = (state: IState) => ({
   display: state.display,
   versions: state.datasetVersions.result.versions,
   versionsLoading: state.datasetVersions.isLoading,
+  tabIndex: state.lineage.tabIndex,
 })
 
 const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
@@ -242,6 +255,7 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
       resetDataset: resetDataset,
       deleteDataset: deleteDataset,
       dialogToggle: dialogToggle,
+      setTabIndex: setTabIndex,
     },
     dispatch
   )