Migrate path translation tests to acceptance tests (#2122)

## Changes The assertions on the output made are now captured in the `output.*` files. These don't capture intent like actual assertions do, but we still have regular test coverage in the path translation tests under `bundle/config/mutator`. ## Tests Tests pass.
databricks · Jan 17, 2025 · 89eb556 · 89eb556
1 parent 9061635
commit 89eb556
Show file tree

Hide file tree

Showing 48 changed files with 314 additions and 142 deletions.
diff --git a/...tests/path_translation/fallback/README.md → acceptance/bundle/paths/fallback/README.md b/...tests/path_translation/fallback/README.md → acceptance/bundle/paths/fallback/README.md
diff --git a/...s/path_translation/nominal/databricks.yml → ...ance/bundle/paths/fallback/databricks.yml b/...s/path_translation/nominal/databricks.yml → ...ance/bundle/paths/fallback/databricks.yml
@@ -1,5 +1,5 @@
 bundle:
-  name: path_translation_nominal
+  name: fallback
 
 include:
   - "resources/*.yml"

diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json
@@ -0,0 +1,67 @@
+[
+  {
+    "job_cluster_key": "default",
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook"
+    },
+    "task_key": "notebook_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
+  },
+  {
+    "dbt_task": {
+      "commands": [
+        "dbt run",
+        "dbt run"
+      ],
+      "project_directory": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/dbt_project"
+    },
+    "job_cluster_key": "default",
+    "task_key": "dbt_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
+    },
+    "task_key": "sql_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "whl": "dist/wheel1.whl"
+      },
+      {
+        "whl": "dist/wheel2.whl"
+      }
+    ],
+    "python_wheel_task": {
+      "package_name": "my_package"
+    },
+    "task_key": "python_wheel_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "jar": "target/jar1.jar"
+      },
+      {
+        "jar": "target/jar2.jar"
+      }
+    ],
+    "spark_jar_task": {
+      "main_class_name": "com.example.Main"
+    },
+    "task_key": "spark_jar_example"
+  }
+]
diff --git a/acceptance/bundle/paths/fallback/output.pipeline.json b/acceptance/bundle/paths/fallback/output.pipeline.json
@@ -0,0 +1,22 @@
+[
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file1.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook1"
+    }
+  },
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file2.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook2"
+    }
+  }
+]
diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt
@@ -0,0 +1,18 @@
+
+>>> $CLI bundle validate -t development -o json
+
+Exit code: 0
+
+>>> $CLI bundle validate -t error
+Error: notebook this value is overridden not found. Local notebook references are expected
+to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
+
+Name: fallback
+Target: error
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/fallback/error
+
+Found 1 error
+
+Exit code: 1
diff --git a/...ath_translation/fallback/override_job.yml → ...ce/bundle/paths/fallback/override_job.yml b/...ath_translation/fallback/override_job.yml → ...ce/bundle/paths/fallback/override_job.yml
diff --git a/...ranslation/fallback/override_pipeline.yml → ...ndle/paths/fallback/override_pipeline.yml b/...ranslation/fallback/override_pipeline.yml → ...ndle/paths/fallback/override_pipeline.yml
diff --git a/...translation/fallback/resources/my_job.yml → ...undle/paths/fallback/resources/my_job.yml b/...translation/fallback/resources/my_job.yml → ...undle/paths/fallback/resources/my_job.yml
@@ -4,33 +4,45 @@ resources:
       name: "placeholder"
       tasks:
         - task_key: notebook_example
+          job_cluster_key: default
           notebook_task:
             notebook_path: "this value is overridden"
 
         - task_key: spark_python_example
+          job_cluster_key: default
           spark_python_task:
             python_file: "this value is overridden"
 
         - task_key: dbt_example
+          job_cluster_key: default
           dbt_task:
             project_directory: "this value is overridden"
             commands:
               - "dbt run"
 
         - task_key: sql_example
+          job_cluster_key: default
           sql_task:
             file:
               path: "this value is overridden"
             warehouse_id: cafef00d
 
         - task_key: python_wheel_example
+          job_cluster_key: default
           python_wheel_task:
             package_name: my_package
           libraries:
             - whl: ../dist/wheel1.whl
 
         - task_key: spark_jar_example
+          job_cluster_key: default
           spark_jar_task:
             main_class_name: com.example.Main
           libraries:
             - jar: ../target/jar1.jar
+
+      # Include a job cluster for completeness
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
diff --git a/...lation/fallback/resources/my_pipeline.yml → .../paths/fallback/resources/my_pipeline.yml b/...lation/fallback/resources/my_pipeline.yml → .../paths/fallback/resources/my_pipeline.yml
diff --git a/acceptance/bundle/paths/fallback/script b/acceptance/bundle/paths/fallback/script
@@ -0,0 +1,10 @@
+errcode trace $CLI bundle validate -t development -o json > output.tmp.json
+
+# Capture job tasks
+jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json
+
+# Capture pipeline libraries
+jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json
+
+# Expect failure for the "error" target
+errcode trace $CLI bundle validate -t error
diff --git a/acceptance/bundle/paths/fallback/script.cleanup b/acceptance/bundle/paths/fallback/script.cleanup
@@ -0,0 +1 @@
+rm -f output.tmp.json
diff --git a/...slation/fallback/src/dbt_project/.gitkeep → ...e/paths/fallback/src/dbt_project/.gitkeep b/...slation/fallback/src/dbt_project/.gitkeep → ...e/paths/fallback/src/dbt_project/.gitkeep
diff --git a/...sts/path_translation/fallback/src/file.py → acceptance/bundle/paths/fallback/src/file.py b/...sts/path_translation/fallback/src/file.py → acceptance/bundle/paths/fallback/src/file.py
diff --git a/...ts/path_translation/fallback/src/file1.py → ...ptance/bundle/paths/fallback/src/file1.py b/...ts/path_translation/fallback/src/file1.py → ...ptance/bundle/paths/fallback/src/file1.py
diff --git a/...ts/path_translation/fallback/src/file2.py → ...ptance/bundle/paths/fallback/src/file2.py b/...ts/path_translation/fallback/src/file2.py → ...ptance/bundle/paths/fallback/src/file2.py
diff --git a/...path_translation/fallback/src/notebook.py → ...nce/bundle/paths/fallback/src/notebook.py b/...path_translation/fallback/src/notebook.py → ...nce/bundle/paths/fallback/src/notebook.py
diff --git a/...ath_translation/fallback/src/notebook1.py → ...ce/bundle/paths/fallback/src/notebook1.py b/...ath_translation/fallback/src/notebook1.py → ...ce/bundle/paths/fallback/src/notebook1.py
diff --git a/...ath_translation/fallback/src/notebook2.py → ...ce/bundle/paths/fallback/src/notebook2.py b/...ath_translation/fallback/src/notebook2.py → ...ce/bundle/paths/fallback/src/notebook2.py
diff --git a/...sts/path_translation/fallback/src/sql.sql → acceptance/bundle/paths/fallback/src/sql.sql b/...sts/path_translation/fallback/src/sql.sql → acceptance/bundle/paths/fallback/src/sql.sql
diff --git a/.../tests/path_translation/nominal/README.md → acceptance/bundle/paths/nominal/README.md b/.../tests/path_translation/nominal/README.md → acceptance/bundle/paths/nominal/README.md
diff --git a/.../path_translation/fallback/databricks.yml → ...tance/bundle/paths/nominal/databricks.yml b/.../path_translation/fallback/databricks.yml → ...tance/bundle/paths/nominal/databricks.yml
@@ -1,5 +1,5 @@
 bundle:
-  name: path_translation_fallback
+  name: nominal
 
 include:
   - "resources/*.yml"

diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json
@@ -0,0 +1,89 @@
+[
+  {
+    "job_cluster_key": "default",
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
+    },
+    "task_key": "notebook_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
+  },
+  {
+    "dbt_task": {
+      "commands": [
+        "dbt run",
+        "dbt run"
+      ],
+      "project_directory": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/dbt_project"
+    },
+    "job_cluster_key": "default",
+    "task_key": "dbt_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
+    },
+    "task_key": "sql_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "whl": "dist/wheel1.whl"
+      },
+      {
+        "whl": "dist/wheel2.whl"
+      }
+    ],
+    "python_wheel_task": {
+      "package_name": "my_package"
+    },
+    "task_key": "python_wheel_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "jar": "target/jar1.jar"
+      },
+      {
+        "jar": "target/jar2.jar"
+      }
+    ],
+    "spark_jar_task": {
+      "main_class_name": "com.example.Main"
+    },
+    "task_key": "spark_jar_example"
+  },
+  {
+    "for_each_task": {
+      "task": {
+        "notebook_task": {
+          "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
+        }
+      }
+    },
+    "job_cluster_key": "default",
+    "task_key": "for_each_notebook_example"
+  },
+  {
+    "for_each_task": {
+      "task": {
+        "job_cluster_key": "default",
+        "spark_python_task": {
+          "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
+        }
+      }
+    },
+    "task_key": "for_each_spark_python_example"
+  }
+]
diff --git a/acceptance/bundle/paths/nominal/output.pipeline.json b/acceptance/bundle/paths/nominal/output.pipeline.json
@@ -0,0 +1,22 @@
+[
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file1.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook1"
+    }
+  },
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file2.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook2"
+    }
+  }
+]
diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt
@@ -0,0 +1,18 @@
+
+>>> $CLI bundle validate -t development -o json
+
+Exit code: 0
+
+>>> $CLI bundle validate -t error
+Error: notebook this value is overridden not found. Local notebook references are expected
+to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
+
+Name: nominal
+Target: error
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/nominal/error
+
+Found 1 error
+
+Exit code: 1
diff --git a/...path_translation/nominal/override_job.yml → ...nce/bundle/paths/nominal/override_job.yml b/...path_translation/nominal/override_job.yml → ...nce/bundle/paths/nominal/override_job.yml
diff --git a/...translation/nominal/override_pipeline.yml → ...undle/paths/nominal/override_pipeline.yml b/...translation/nominal/override_pipeline.yml → ...undle/paths/nominal/override_pipeline.yml
diff --git a/..._translation/nominal/resources/my_job.yml → ...bundle/paths/nominal/resources/my_job.yml b/..._translation/nominal/resources/my_job.yml → ...bundle/paths/nominal/resources/my_job.yml
@@ -4,38 +4,45 @@ resources:
       name: "placeholder"
       tasks:
         - task_key: notebook_example
+          job_cluster_key: default
           notebook_task:
             notebook_path: "this value is overridden"
 
         - task_key: spark_python_example
+          job_cluster_key: default
           spark_python_task:
             python_file: "this value is overridden"
 
         - task_key: dbt_example
+          job_cluster_key: default
           dbt_task:
             project_directory: "this value is overridden"
             commands:
               - "dbt run"
 
         - task_key: sql_example
+          job_cluster_key: default
           sql_task:
             file:
               path: "this value is overridden"
             warehouse_id: cafef00d
 
         - task_key: python_wheel_example
+          job_cluster_key: default
           python_wheel_task:
             package_name: my_package
           libraries:
             - whl: ../dist/wheel1.whl
 
         - task_key: spark_jar_example
+          job_cluster_key: default
           spark_jar_task:
             main_class_name: com.example.Main
           libraries:
             - jar: ../target/jar1.jar
 
         - task_key: for_each_notebook_example
+          job_cluster_key: default
           for_each_task:
             task:
               notebook_task:
@@ -44,5 +51,12 @@ resources:
         - task_key: for_each_spark_python_example
           for_each_task:
             task:
+              job_cluster_key: default
               spark_python_task:
                 python_file: "this value is overridden"
+
+      # Include a job cluster for completeness
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
diff --git a/...slation/nominal/resources/my_pipeline.yml → ...e/paths/nominal/resources/my_pipeline.yml b/...slation/nominal/resources/my_pipeline.yml → ...e/paths/nominal/resources/my_pipeline.yml
diff --git a/acceptance/bundle/paths/nominal/script b/acceptance/bundle/paths/nominal/script
@@ -0,0 +1,10 @@
+errcode trace $CLI bundle validate -t development -o json > output.tmp.json
+
+# Capture job tasks
+jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json
+
+# Capture pipeline libraries
+jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json
+
+# Expect failure for the "error" target
+errcode trace $CLI bundle validate -t error
diff --git a/acceptance/bundle/paths/nominal/script.cleanup b/acceptance/bundle/paths/nominal/script.cleanup
@@ -0,0 +1 @@
+rm -f output.tmp.json
diff --git a/...nslation/nominal/src/dbt_project/.gitkeep → ...le/paths/nominal/src/dbt_project/.gitkeep b/...nslation/nominal/src/dbt_project/.gitkeep → ...le/paths/nominal/src/dbt_project/.gitkeep
diff --git a/...ests/path_translation/nominal/src/file.py → acceptance/bundle/paths/nominal/src/file.py b/...ests/path_translation/nominal/src/file.py → acceptance/bundle/paths/nominal/src/file.py
diff --git a/...sts/path_translation/nominal/src/file1.py → acceptance/bundle/paths/nominal/src/file1.py b/...sts/path_translation/nominal/src/file1.py → acceptance/bundle/paths/nominal/src/file1.py
diff --git a/...sts/path_translation/nominal/src/file2.py → acceptance/bundle/paths/nominal/src/file2.py b/...sts/path_translation/nominal/src/file2.py → acceptance/bundle/paths/nominal/src/file2.py
diff --git a/.../path_translation/nominal/src/notebook.py → ...ance/bundle/paths/nominal/src/notebook.py b/.../path_translation/nominal/src/notebook.py → ...ance/bundle/paths/nominal/src/notebook.py
diff --git a/...path_translation/nominal/src/notebook1.py → ...nce/bundle/paths/nominal/src/notebook1.py b/...path_translation/nominal/src/notebook1.py → ...nce/bundle/paths/nominal/src/notebook1.py
diff --git a/...path_translation/nominal/src/notebook2.py → ...nce/bundle/paths/nominal/src/notebook2.py b/...path_translation/nominal/src/notebook2.py → ...nce/bundle/paths/nominal/src/notebook2.py
diff --git a/...ests/path_translation/nominal/src/sql.sql → acceptance/bundle/paths/nominal/src/sql.sql b/...ests/path_translation/nominal/src/sql.sql → acceptance/bundle/paths/nominal/src/sql.sql
diff --git a/.../relative_path_translation/databricks.yml → .../relative_path_translation/databricks.yml b/.../relative_path_translation/databricks.yml → .../relative_path_translation/databricks.yml
diff --git a/acceptance/bundle/paths/relative_path_translation/output.default.json b/acceptance/bundle/paths/relative_path_translation/output.default.json
@@ -0,0 +1,6 @@
+{
+  "paths": [
+    "/Workspace/remote/src/file1.py",
+    "/Workspace/remote/src/file1.py"
+  ]
+}