use recipe for kpdb (#1301)

.github/workflows/build.yml

@@ -210,6 +210,8 @@ jobs:
     with:
       image_tag: ${{ needs.health_check.outputs.tag }}
       build_name: ${{ needs.health_check.outputs.build_name }}
+      recipe_file: ${{ inputs.recipe_file }}
+      plan_command: ${{ needs.health_check.outputs.plan_command }}
       dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   pluto:
     needs: health_check

.github/workflows/knownprojects_build.yml

@@ -9,6 +9,15 @@ on:
       build_name:
         type: string
         required: true
+      recipe_file:
+        type: string
+        required: true
+      version:
+        type: string
+        required: false
+      plan_command:
+        type: string
+        default: recipe
       dev_bucket:
         type: string
         required: false
@@ -60,23 +69,33 @@ jobs:
           dbt deps
           dbt debug
 
-      - name: dataloading ..
+      - name: Plan build
+        run: python3 -m dcpy.lifecycle.builds.plan ${{ inputs.plan_command }}
+
+      - name: Set recipe env vars
+        working-directory: ./
+        run: source ./bash/export_recipe_env.sh products/knownprojects/${{ inputs.recipe_file }}.lock.yml
+
+      - name: Dataloading
+        run: python -m dcpy.lifecycle.builds.load load --recipe-path ${{ inputs.recipe_file }}.lock.yml
+
+      - name: Non-recipe dataloading
         run: ./bash/01_dataloading.sh
 
-      - name: build ...
+      - name: Build
         run: ./bash/02_build.sh
 
-      - name: test build tables ...
+      - name: Test build tables
         run: dbt test --select kpdb
 
-      - name: aggregate ...
+      - name: Aggregate
         run: ./bash/03_aggregate.sh
 
-      - name: test aggregate tables ...
+      - name: Test aggregate tables
         run: dbt test --select aggregate
 
-      - name: export ...
+      - name: Export
         run: ./bash/04_export.sh
 
-      - name: upload ...
+      - name: Upload
         run: python3 -m python.upload

products/knownprojects/bash/01_dataloading.sh

@@ -1,24 +1,11 @@
 #!/bin/bash
 source ../../bash/utils.sh
 set_error_traps
-max_bg_procs 5
-
-if [ -n "${BUILD_ENGINE_SCHEMA}" ]; then
-    echo "Dropping and creating build schema '${BUILD_ENGINE_SCHEMA}'"
-    run_sql_command "DROP SCHEMA IF EXISTS ${BUILD_ENGINE_SCHEMA} CASCADE;"
-    echo "Dropping build tests schema '${BUILD_ENGINE_SCHEMA_TESTS}'"
-    run_sql_command "DROP SCHEMA IF EXISTS ${BUILD_ENGINE_SCHEMA_TESTS} CASCADE;"
-
-    run_sql_command "VACUUM (ANALYZE);"
-    run_sql_command "CREATE SCHEMA ${BUILD_ENGINE_SCHEMA};"
-fi
 
 # Load source data
 rm -rf data
 mkdir -p data
 
-create_source_data_table
-
 # download data/raw
 # download data/corrections
 python3 -m python.download
@@ -38,33 +25,6 @@ python3 -m python.extractors dcp_knownprojects
 # Load corrections tables
 run_sql_file sql/create_corrections.sql
 
-# Load ZAP tables
-# * Versions pinned for Housing team's Text Amendment model
-zap_version=20230905
-import_recipe dcp_projects ${zap_version}
-import_recipe dcp_projectactions ${zap_version}
-import_recipe dcp_projectbbls ${zap_version}
-import_recipe dcp_dcpprojectteams ${zap_version}
-
-# Load other tables
-# * Versions pinned for Housing team's Text Amendment model
-housingdb_verion=23Q2
-import_recipe dcp_mappluto_wi
-import_recipe dcp_boroboundaries
-import_recipe dcp_housing ${housingdb_verion}
-import_recipe dcp_zoningmapamendments
-
-# Load SCA Geometry Aggregate Tables
-import_recipe doe_eszones
-import_recipe doe_school_subdistricts
-import_recipe dcp_school_districts
-
-# Load geographic boundaries Aggregate Tables
-import_recipe dcp_ct2020_wi
-import_recipe dcp_nta2020
-import_recipe dcp_cdta2020
-import_recipe dcp_cdboundaries_wi
-
 echo
 echo "data loading complate"
 echo

products/knownprojects/recipe.yml

@@ -0,0 +1,26 @@
+name: Known Projects Database
+product: db-kpdb
+vars:
+  ZAP_VERSION: "20230905"
+inputs:
+  missing_versions_strategy: find_latest
+  datasets:
+  - name: dcp_projects
+    version_env_var: ZAP_VERSION
+  - name: dcp_projectactions
+    version_env_var: ZAP_VERSION
+  - name: dcp_projectbbls
+    version_env_var: ZAP_VERSION
+  - name: dcp_dcpprojectteams
+    version_env_var: ZAP_VERSION
+  - name: dcp_mappluto_wi
+  - name: dcp_boroboundaries
+  - name: dcp_housing
+  - name: dcp_zoningmapamendments
+  - name: doe_eszones
+  - name: doe_school_subdistricts
+  - name: dcp_school_districts
+  - name: dcp_ct2020_wi
+  - name: dcp_nta2020
+  - name: dcp_cdta2020
+  - name: dcp_cdboundaries_wi