diff --git a/.github/workflows/hamilton-ui-backend.yml b/.github/workflows/hamilton-ui-backend.yml
new file mode 100644
index 000000000..66232e3f0
--- /dev/null
+++ b/.github/workflows/hamilton-ui-backend.yml
@@ -0,0 +1,83 @@
+name: Backend Test Workflow
+
+on:
+ push:
+ branches:
+ - main # or any specific branches you want to include
+ paths:
+ - 'ui/backend/**'
+
+ pull_request:
+ branches:
+ - main
+ paths:
+ - 'ui/backend/**'
+
+
+jobs:
+ test-backend:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ testdir: [test_lifecycle, test_db_methods] # Specify your test directories
+ services:
+ postgres:
+ image: postgres:14
+ env:
+ POSTGRES_USER: postgres
+ POSTGRES_DB: circleci_test
+ POSTGRES_HOST_AUTH_METHOD: trust
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 100s
+ --health-retries 10
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.9'
+ - name: Install dependencies
+ run: |
+ cd ui/backend/server
+ pip install -r requirements-base.txt
+ pip install -r requirements-test.txt
+ - name: Run migrations
+ env:
+ DB_HOST: localhost
+ DB_USER: postgres
+ DB_PASSWORD: "postgres"
+ DB_NAME: ${{ matrix.testdir }}
+ DB_PORT: 5432
+ HAMILTON_ENV: integration_tests
+ DJANGO_SECRET_KEY: test
+ PGPASSWORD: postgres
+ PGHOST: localhost
+ PGUSER: postgres
+ HAMILTON_BLOB_STORE: local
+ HAMILTON_LOCAL_BLOB_DIR: ./blob_data
+ run: |
+ cd ui/backend/server
+ python manage.py sqlcreate
+ echo $(python manage.py sqlcreate) | psql -U postgres
+ python manage.py migrate
+ - name: Run tests
+ env:
+ DB_HOST: localhost
+ DB_USER: postgres
+ DB_PASSWORD: "postgres"
+ DB_NAME: ${{ matrix.testdir }}
+ DB_PORT: 5432
+ HAMILTON_ENV: integration_tests
+ DJANGO_SECRET_KEY: test
+ PGPASSWORD: postgres
+ PGHOST: localhost
+ PGUSER: postgres
+ HAMILTON_BLOB_STORE: local
+ HAMILTON_LOCAL_BLOB_DIR: ./blob_data
+ run: |
+ cd ui/backend/server
+ python -m pytest tests/${{ matrix.testdir }} -vvvvv
diff --git a/.github/workflows/hamilton-ui-frontend.yml b/.github/workflows/hamilton-ui-frontend.yml
new file mode 100644
index 000000000..019898825
--- /dev/null
+++ b/.github/workflows/hamilton-ui-frontend.yml
@@ -0,0 +1,30 @@
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - 'ui/frontend/**'
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths:
+ - 'ui/frontend/**'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: ui/frontend
+ strategy:
+ matrix:
+ node-version: [16.x]
+ steps:
+ - uses: actions/checkout@v3
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v1
+ with:
+ node-version: ${{ matrix.node-version }}
+ - run: npm install --ignore-scripts
+ - run: npm run lint-fix
+ - run: npm run format
+ - run: npm run build
diff --git a/LICENSE b/LICENSE
index ca261c967..558bd0f1c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,5 +1,11 @@
-Copyright (c) 2019-2022 Stitch Fix
-Copyright (c) 2023-2023 DAGWorks Inc.
+Portions of this software are licensed as follows:
+
+* All content that resides under the "ui/frontend/src/ee" and "ui/backend/server/trackingserver_auth" directory of this
+repository is licensed under the license defined in "ui/frontend/src/ee/LICENSE" "ui/backend/server/trackingserver_auth/LICENSE" respectively.
+* All third party components incorporated into the Hamilton Software are licensed under the original license provided by the owner of the applicable component.
+* Content outside of the above mentioned directories or restrictions above is available under the "BSD-3 Clear Clause" license as defined below.
+
+Copyright (c) 2023-2024 DAGWorks Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted
@@ -9,7 +15,7 @@ Redistribution and use in source and binary forms, with or without modification,
disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided with the distribution.
-* Neither the name of Stitch Fix, DAGWorks, nor the names of its contributors may be used to endorse or promote products derived
+* Neither the name of DAGWorks Inc., nor the names of its contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY
diff --git a/README.md b/README.md
index 890f17ac7..2d195dd42 100644
--- a/README.md
+++ b/README.md
@@ -432,6 +432,8 @@ If you do not wish to participate, one can opt-out with one of the following met
HAMILTON_TELEMETRY_ENABLED=false python NAME_OF_MY_DRIVER.py
```
+For the hamilton UI you jmust use the environment variable method prior to running docker compose.
+
# Contributors
## Code Contributors
diff --git a/docs/_static/catalog.png b/docs/_static/catalog.png
new file mode 100644
index 000000000..b4823b851
Binary files /dev/null and b/docs/_static/catalog.png differ
diff --git a/docs/_static/code_browser.png b/docs/_static/code_browser.png
new file mode 100644
index 000000000..436aebe09
Binary files /dev/null and b/docs/_static/code_browser.png differ
diff --git a/docs/_static/dag_view.png b/docs/_static/dag_view.png
new file mode 100644
index 000000000..20e2db8c6
Binary files /dev/null and b/docs/_static/dag_view.png differ
diff --git a/docs/_static/new_project.png b/docs/_static/new_project.png
new file mode 100644
index 000000000..a78fadbad
Binary files /dev/null and b/docs/_static/new_project.png differ
diff --git a/docs/_static/run_data.png b/docs/_static/run_data.png
new file mode 100644
index 000000000..82a0524c6
Binary files /dev/null and b/docs/_static/run_data.png differ
diff --git a/docs/_static/run_telemetry.png b/docs/_static/run_telemetry.png
new file mode 100644
index 000000000..995d8b5e3
Binary files /dev/null and b/docs/_static/run_telemetry.png differ
diff --git a/docs/_static/run_tracking.png b/docs/_static/run_tracking.png
new file mode 100644
index 000000000..f48ae47dd
Binary files /dev/null and b/docs/_static/run_tracking.png differ
diff --git a/docs/_static/version_tracking.png b/docs/_static/version_tracking.png
new file mode 100644
index 000000000..311c1b274
Binary files /dev/null and b/docs/_static/version_tracking.png differ
diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst
index 16f4d008f..80d75ddea 100644
--- a/docs/concepts/index.rst
+++ b/docs/concepts/index.rst
@@ -17,4 +17,5 @@ concepts that makes Hamilton unique and powerful.
builder
function-modifiers-advanced
parallel-task
+ ui
best-practices/index
diff --git a/docs/concepts/ui.rst b/docs/concepts/ui.rst
new file mode 100644
index 000000000..97d5c4e0e
--- /dev/null
+++ b/docs/concepts/ui.rst
@@ -0,0 +1,199 @@
+===========
+Hamilton UI
+===========
+
+Hamilton comes with a fully open-source UI that can be run both for local deployment and on a remote server.
+The UI consists of the following features:
+
+1. Telemetry for hamilton executions -- both on the history of executions and the data itself
+2. A feature/artifact catalog for browsing/connecting executions of nodes -> results
+3. A DAG visualizer for exploring the DAG, looking at the code, and determining lineage
+
+--------
+Overview
+--------
+
+Getting Started
+---------------
+
+The Hamilton UI is contained within a set of docker images. You launch with `docker-compose
+ Login +
++ Sign up +
++ Dashboard +
++ {props.dataTypeName} + | + } + {props.columns.map((column, index) => { + return ( ++ {column.displayName} + | + ); + })} +
---|
+ { + | + {props.dataTypeName} + | + } + {props.columns.map((column, index) => { + return ( ++ {column.displayName} + | + ); + })} +
---|
+ {parsePythonType(pythonType)}
+
+ );
+ // return {props.nodeName};
+ },
+ },
+ {
+ displayName: "State",
+ Render: (nodeRun: NodeRunAndTemplate) => {
+ // TODO -- ensure that this is the right type
+ let status = nodeRun.status as RunStatusType;
+ status = adjustStatusForDuration(
+ status,
+ nodeRun.start_time || undefined,
+ nodeRun.end_time || undefined,
+ new Date()
+ );
+ return { + setExpanded(!expanded); + e.stopPropagation(); + }} + className={`${ + expanded ? "break-word whitespace-pre-wrap" : "truncate" + } text-gray-500 cursor-cell`} + > + {props.documentation} ++
;
+};
diff --git a/ui/frontend/src/components/dashboard/Code/CodeViewUtils.tsx b/ui/frontend/src/components/dashboard/Code/CodeViewUtils.tsx
new file mode 100644
index 000000000..a28cfc146
--- /dev/null
+++ b/ui/frontend/src/components/dashboard/Code/CodeViewUtils.tsx
@@ -0,0 +1,26 @@
+import { NodeTemplate } from "../../../state/api/friendlyApi";
+import { getPythonTypeIconFromNode } from "../../../utils";
+
+export type NodeType = "input" | "intermediate" | "output";
+const colors = {
+ output: "bg-dwdarkblue",
+ input: "bg-dwdarkblue/50",
+ intermediate: "bg-dwdarkblue",
+};
+
+export const NodeView: React.FC<{
+ node: NodeTemplate;
+ type: NodeType;
+}> = ({ node, type }) => {
+ const color = colors[type];
+ const Icon = getPythonTypeIconFromNode(node);
+ return (
+ + {tokens.map((line, i) => ( + // eslint-disable-next-line react/jsx-key ++ ); + }} ++ {line.map((token, key) => ( + // eslint-disable-next-line react/jsx-key + + ))} ++ ))} +
+
+ Modify your regular Hamilton driver to log to the Hamilton UI: +
++ Run your DAG. A new version will show up! Note that the versions are + keyed on the "dag_name" as well as the code in the + modules. you pass. No new version will be logged if neither of those + change. +
++ Initialize the project from one of our demo project templates: +
++ Install python dependencies for the example: +
+Run it!
++ + Track your runs! + +
+
+ Get an API key from the left side menu (
+
+
+ Install the Hamilton SDK: +
++ Track execution of DAGs, visualize your pipelines, and understand + how they change over time! +
+ ++ Enter emails or select teams you are a part of. +
++ Enter emails or select teams you are a part of. +
++ ❗{error} +
+ ))} ++ {props.variableType} + | + {props.runs.map((run, i) => ( +
+ |
+ ))}
+
---|---|
+ {variableKey}
+ |
+ {values.map((value, i) => {
+ return (
+ {
+ props.setHighlightedRun(
+ props.runs[i].id as number
+ );
+ }}
+ onMouseLeave={() => {
+ props.setHighlightedRun(null);
+ }}
+ key={i}
+ className={`truncate break-words px-3 py-4 text-sm ${
+ hasDiff
+ ? "bg-yellow-400 text-gray-700"
+ : "text-gray-500"
+ } ${
+ hasDiff &&
+ props.highlightedRun === props.runs[i].id
+ ? "bg-yellow-400/50"
+ : hasDiff
+ ? "bg-yellow-400"
+ : props.highlightedRun === props.runs[i].id
+ ? "bg-dwdarkblue/5"
+ : ""
+ }`}
+ >
+
+
+
+ {value?.toString()}
+
+ |
+ );
+ })}
+
+ {type}
+
+ );
+ })}
+
+
+
+ |
+ {cols(hasAnyGroupingTask).map((col, index) => (
+ + {col.displayName} + | + ))} +
---|---|
+ {/* |
+ {allCols.map((col, index) => (
+ + {col.display} + | + ))} + {} + |
---|
+ {error.value.stack_trace.join("\n")} ++ ) : ( +
{currentTask?.node_name}
produced no errors!{" "}
+ {item}
+ ))}
+ true
,
+ Render: (
+ value: (DwDescribeV003BooleanColumnStatistics & { runId: number })[],
+ isSummaryRow: boolean,
+ isExpanded: boolean
+ ) => {
+ const numericValues = value
+ .map((item) => item.zeros)
+ .filter((item) => item !== undefined) as number[];
+ if (!isSummaryRow) {
+ return false
,
+ Render: (
+ value: (DwDescribeV003BooleanColumnStatistics & { runId: number })[],
+ isSummaryRow: boolean,
+ isExpanded: boolean
+ ) => {
+ const numericValues = value
+ .map((item) => item.count - item.zeros)
+ .filter((item) => item !== undefined) as number[];
+
+ if (!isSummaryRow) {
+ return ({topCount})
+
+ {meanStr === undefined || stdStr === undefined
+ ? ""
+ : `${meanStr}±${stdStr}`}
+
+
+ [{truncateAndRemoveTrailingZeroes(minValue, 3)},{" "}
+ {truncateAndRemoveTrailingZeroes(maxValue, 3)}]
+
+ );
+ },
+ },
+ {
+ displayName: histogram
,
+ Render: (
+ value: (DwDescribeV003NumericColumnStatistics & { runId: number })[],
+ isSummaryRow: boolean,
+ isExpanded: boolean,
+ index: number | undefined
+ ) => {
+ const histograms = value.map((item) => item.histogram);
+ if (isSummaryRow && isExpanded) {
+ return <>>;
+ }
+ return (
+ quantiles
,
+ Render: (
+ value: (DwDescribeV003NumericColumnStatistics & { runId: number })[],
+ isSummaryRow: boolean,
+ isExpanded: boolean,
+ index: number | undefined
+ ) => {
+ const quantiles = value.map((item) => item.quantiles);
+ if (isSummaryRow && isExpanded) {
+ return <>>;
+ }
+ return (
+
+ [{minValue.toDateString()}, {maxValue.toDateString()}]
+
+ );
+ },
+ },
+ {
+ displayName: histogram
,
+ Render: (
+ value: (DwDescribeV003NumericColumnStatistics & { runId: number })[],
+ isSummaryRow: boolean,
+ isExpanded: boolean,
+ index: number | undefined
+ ) => {
+ const histograms = value.map((item) => item.histogram);
+ if (isSummaryRow && isExpanded) {
+ return <>>;
+ }
+ return (
+ quantiles
,
+ Render: (
+ value: (DwDescribeV003NumericColumnStatistics & { runId: number })[],
+ isSummaryRow: boolean,
+ isExpanded: boolean,
+ index: number | undefined
+ ) => {
+ const quantiles = value.map((item) => item.quantiles);
+ if (isSummaryRow && isExpanded) {
+ return <>>;
+ }
+ return (
+
+ {parsePythonType({ type_name: item.type })}
+
+ { + setExpanded(!expanded); + e.stopPropagation(); + }} + className={`${ + expanded ? "break-word whitespace-pre-wrap" : "truncate" + } text-gray-500 cursor-cell`} + > + {item.value.toString()} ++ {/*
+ {item.value.toString()}
+
*/}
+ [{props.runIds.join(", ")}]
+ for {props.taskName}
. We are working on adding support for
+ everything -- reach out if you need it and we can prioritize!
+
+ {item.unsupported_type}
+
+ // {parsePythonType({ typeName: item.type })}
+ //
+ // { + // setExpanded(!expanded); + // e.stopPropagation(); + // }} + // className={`${ + // expanded ? "break-word whitespace-pre-wrap" : "truncate" + // } text-gray-500 cursor-cell`} + // > + // {item.value.toString()} + //+ // {/*
+ // {item.value.toString()}
+ //
*/}
+ // [{runIds.join(", ")}]
+ // for {task_name}
. We are working on adding support for
+ // everything -- reach out if you need it and we can prioritize!
+ //
+ // {item.unsupported_type}
+ // {value[0].top?.toString() || ""}; + } + return <>>; + }, + }, + { + displayName: "freq", + Render: ( + value: PandasDescribeCategoricalColumn[], + isSummaryRow: boolean + ) => { + if (!isSummaryRow) { + return ( +
{value.key_start ? `${value.key_start}...` : ""}
+ ),
+ },
+ {
+ displayName: "",
+ Render: (value: ApiKeyOut) => (
+ {props.version.code_hash}
}
+
+ {props.version.dag_hash}
}
+
+ + {col.display} + | + ))} +
---|
+ {tokens.map((line, i) => ( + // eslint-disable-next-line react/jsx-key ++ ); + }} ++ {line.map((token, key) => ( + // eslint-disable-next-line react/jsx-key + + ))} ++ ))} +
{fn?.contents}*/} +