Merge pull request #35 from fivetran/feature/disable-exchange-rate-va…

…riable [Feature] Add disable exchange rate variable
fivetran · Jul 26, 2023 · 546cc57 · 546cc57
2 parents 4311576 + 29373af
commit 546cc57
Show file tree

Hide file tree

Showing 15 changed files with 38 additions and 24 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -20,7 +20,7 @@ dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 ## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE
-dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
+dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 ### END VARS CHUNK, REMOVE IF NOT USING
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,18 @@
-# dbt_netsuite_source v0.UPDATE.UPDATE
+# dbt_netsuite_source v0.6.3
+[PR #35](https://github.com/fivetran/dbt_netsuite_source/pull/35) applies the following changes:
 
- ## Under the Hood:
+## 🎉 Feature Updates 🎉
+- Introduces variable `netsuite2__using_exchange_rate` to allow users who don't utilize exchange rates in Netsuite2 the ability to disable that functionality within your `dbt_project.yml`, since these models aren't being used.
+- **Note**: If you are using the `dbt_netsuite` package, be sure to set it globally by inserting this code into your `dbt_project.yml`:
+```yml
+vars:
+  netsuite2__using_exchange_rate: false
+```
 
+## Under the Hood:
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
+
 # dbt_netsuite_source v0.6.2
 
 ## 🎉 Feature Updates 🎉

diff --git a/README.md b/README.md
@@ -113,8 +113,10 @@ It's possible that your Netsuite connector does not sync every table that this p
 ```yml
 vars:
     netsuite2__multibook_accounting_enabled: false # True by default. Disable `accountingbooksubsidiary` and `accountingbook` if you are not using the Multi-Book Accounting feature
+    netsuite2__using_exchange_rate: false #True by default. Disable `exchange_rate` if you don't utilize exchange rates. If you set this variable to false and are using the `netsuite` transformation package, ensure it is scoped globally so that the `netsuite` package can access it as well.  
     netsuite2__using_vendor_categories: false # True by default. Disable `vendorcategory` if you don't categorize your vendors
     netsuite2__using_jobs: false # True by default. Disable `job` if you don't use jobs
+
 ```
 > **Note**: The Netsuite dbt package currently only supports disabling transforms of [Multi-Book Accounting](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/book_3831565332.html) related tables (`accountingbooksubsidiary` and `accountingbook`), the `vendorcategory` source table, and the `job` source table. Please create an issue to request additional tables and/or [features](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/bridgehead_N233872.html) to exclude. 
 > 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.6.2'
+version: '0.6.3'
 
 models:
   netsuite_source:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore
@@ -1,3 +1,4 @@
 target/
 dbt_modules/
-logs/
+logs/
+env/
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: netsuite_source_integration_tests_1
+      schema: netsuite_source_integration_tests_2
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: netsuite_source_integration_tests_1
+      schema: netsuite_source_integration_tests_2
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: netsuite_source_integration_tests_1
+      schema: netsuite_source_integration_tests_2
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: netsuite_source_integration_tests_1
+      schema: netsuite_source_integration_tests_2
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: netsuite_source_integration_tests_1
-      threads: 2
+      schema: netsuite_source_integration_tests_2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,12 +1,12 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.6.2'
+version: '0.6.3'
 
 profile: 'integration_tests'
 config-version: 2
 vars:
   netsuite_source:
-    netsuite_schema: netsuite_source_integration_tests_1
+    netsuite_schema: netsuite_source_integration_tests_2
     netsuite_data_model_override: netsuite
     netsuite_accounting_books_identifier: "netsuite_accounting_books_data"
     netsuite_accounting_periods_identifier: "netsuite_accounting_periods_data"

diff --git a/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql b/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }}
+{{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__using_exchange_rate', true))) }}
 
 with base as (
 

diff --git a/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }}
+{{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__using_exchange_rate', true))) }}
 
 select *
 from {{ var('netsuite2_consolidated_exchange_rates') }}