Make local execution easier

.gitignore

@@ -31,3 +31,6 @@ __pycache__/
 
 # jetbrains
 .idea
+
+# Metaf;ow
+.metaflow

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "metaflow-ui"]
+	path = metaflow-ui
+	url = https://github.com/Netflix/metaflow-ui

docker-compose.development.yml

@@ -15,6 +15,7 @@ services:
     cap_add:
       - SYS_PTRACE
     environment:
+      - MF_KUBERNETES_NAMESPACE=argo
       - MF_METADATA_DB_HOST=db
       - MF_METADATA_DB_PORT=5432
       - MF_METADATA_DB_USER=postgres
@@ -77,6 +78,7 @@ services:
     volumes:
       - ./services:/root/services
     environment:
+      - METAFLOW_KUBERNETES_NAMESPACE=argo
       - MF_METADATA_DB_HOST=db
       - MF_METADATA_DB_PORT=5432
       - MF_METADATA_DB_USER=postgres
@@ -88,6 +90,17 @@ services:
       - MF_MIGRATION_PORT=${MF_MIGRATION_PORT:-8082}
     depends_on:
       - db
+  metaflow-ui:
+    build:
+      context: ./metaflow-ui
+      dockerfile: Dockerfile
+    ports:
+      - "3000:3000"
+    environment:
+      - METAFLOW_SERVICE=http://localhost:8083/
+      - METAFLOW_KUBERNETES_NAMESPACE=argo
+    depends_on:
+      - ui_backend
   db:
     image: "postgres:11"
     command: ["postgres", "-c", "log_statement=none", "-c", "wal_level=logical"]

local-example/README.md

@@ -0,0 +1,23 @@
+# Running Flows in Local UI
+
+1. Start your metadata services using the following command:
+
+    ```bash
+    ./local-example/run-services.sh
+    ```
+
+2. Configure metaflow to use your local metadata services:
+
+    ```bash
+    metaflow configure import local-example/metaflow-config.json
+    ```
+3. 
+
+4. Run the example flow:
+
+    ```bash
+    python local-example/helloworld.py run
+    ```
+5. Go to your UI metaflow UI at http://localhost:3000/ to see your flow runs.
+
+![](../../../../Downloads/metaflow UI.png)

local-example/helloworld.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+from metaflow import FlowSpec, step, conda, kubernetes
+
+
+class HelloFlow(FlowSpec):
+    """
+    A flow where Metaflow prints 'Hi'.
+
+    Run this flow to validate that Metaflow is installed correctly.
+
+    """
+    @step
+    def start(self):
+        """
+        This is the 'start' step. All flows must have a step named 'start' that
+        is the first step in the flow.
+
+        """
+        print("HelloFlow is starting.")
+        self.next(self.hello)
+
+    @kubernetes(cpu=.3, memory=4, namespace="argo")
+    @step
+    def hello(self):
+        """
+        A step for metaflow to introduce itself.
+
+        """
+        print("Metaflow says: Hi!")
+        self.next(self.end)
+
+    @step
+    def end(self):
+        """
+        This is the 'end' step. All flows must have an 'end' step, which is the
+        last step in the flow.
+
+        """
+        print("HelloFlow is all done.")
+
+
+if __name__ == '__main__':
+    HelloFlow()

local-example/metaflow-config.json

@@ -0,0 +1,8 @@
+{
+    "METAFLOW_SERVICE_URL": "http://localhost:8080",
+    "METAFLOW_DEFAULT_METADATA": "service",
+    "METAFLOW_DEFAULT_DATASTORE": "s3",
+    "METAFLOW_DATASTORE_SYSROOT_S3": "s3://your-bucket",
+    "METAFLOW_DATATOOLS_S3ROOT": "s3://your-bucket",
+    "METAFLOW_KUBERNETES_NAMESPACE": "argo"
+}

local-example/run_services.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+docker-compose -f docker-compose.development.yml up -d --build &
+
+# Mac installation steps
+#brew install minikube
+#brew install argo
+#brew install kubectl
+
+
+pip install kubernetes
+echo "Starting Minikube"
+minikube start
+
+alias kubectl="minikube kubectl --"
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.1.1/cert-manager.yaml
+kubectl create ns argo
+kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml
+kubectl -n argo port-forward deployment/argo-server 2746:2746
+
+wait