Merge pull request #51 from FrankTub/feature/enable-entity-pass-throu…

…gh-columns

Feature #50: Enable teams to configure additional columns for staging entities models

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -83,6 +83,6 @@ body:
       description: Our team will assess this issue and let you know if we will add it to a future sprint. However, if you would like to expedite the solution, we encourage you to contribute to the package via a PR. Our team will then work with you to approve and merge your contributions as soon as possible.
       options:
         - label: Yes.
-        - label: Yes, but I will need assistance and will schedule time during our [office hours](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) for guidance
+        - label: Yes, but I will probably need assistance.
         - label: No.
           required: false

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -34,7 +34,7 @@ body:
       description: Our team will assess this feature and let you know if we will add it to a future sprint. However, if you would like to expedite the feature, we encourage you to contribute to the package via a PR. Our team will then work with you to approve and merge your contributions as soon as possible.
       options:
         - label: Yes.
-        - label: Yes, but I will need assistance and will schedule time during your [office hours](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) for guidance.
+        - label: Yes, but I will probably need assistance.
         - label: No.
           required: false
   - type: textarea

.gitignore

@@ -5,5 +5,6 @@ logs/
 keyfile.json
 .DS_Store
 develop/
-
-dbt_packages/
+env/
+dbt_packages/
+package-lock.yml

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_netsuite_source v0.10.1
+[PR #51](https://github.com/fivetran/dbt_netsuite_source/pull/51) includes the following update:
+
+## Feature Update
+- Added the `entities_pass_through_columns` variable to be leveraged within the `stg_netsuite2__entities` model. This variable will allow users to pass through fields which are not natively included to the package staging model.
+  - Removed fields from the `get_entity_columns` macro that are not brought into the `stg_netsuite2__entities` model to ensure that the passthrough feature works properly.
+
+## Contributors:
+- [@FrankTub](https://github.com/FrankTub) ([PR #51](https://github.com/fivetran/dbt_netsuite_source/pull/51))
+
 # dbt_netsuite_source v0.10.0
 [PR #47](https://github.com/fivetran/dbt_netsuite_source/pull/47) includes the following update:
 ## 🚨 Breaking Changes 🚨

README.md

@@ -148,6 +148,9 @@ vars:
     customers_pass_through_columns: 
         - name: "customer_custom_field"
           alias: "customer_field"
+    entities_pass_through_columns:
+        - name: "entity_custom_field"
+          alias: "entity_field"
     locations_pass_through_columns: 
         - name: "location_custom_field"
     subsidiaries_pass_through_columns: 
@@ -222,5 +225,4 @@ We highly encourage and welcome contributions to this package. Check out [this d
 
 # 🏪 Are there any resources available?
 - If you have questions or want to reach out for help, please refer to the [GitHub Issue](https://github.com/fivetran/dbt_netsuite_source/issues/new/choose) section to find the right avenue of support for you.
-- If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW).
-- Have questions or want to be part of the community discourse? Create a post in the [Fivetran community](https://community.fivetran.com/t5/user-group-for-dbt/gh-p/dbt-user-group) and our team along with the community can join in on the discussion!
+- If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW).

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.10.0'
+version: '0.10.1'
 
 models:
   netsuite_source:
@@ -67,6 +67,7 @@ vars:
     consolidated_exchange_rates_pass_through_columns: []
     vendors_pass_through_columns: []
     items_pass_through_columns: []
+    entities_pass_through_columns: []
 
 clean-targets:
 - target

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.10.0'
+version: '0.10.1'
 
 profile: 'integration_tests'
 config-version: 2

integration_tests/requirements.txt

@@ -1,7 +1,7 @@
-dbt-snowflake>=1.3.0,<2.0.0
-dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<2.0.0
-dbt-postgres>=1.3.0,<2.0.0
-dbt-spark>=1.3.0,<2.0.0
-dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.6.0,<2.0.0
+dbt-snowflake>=1.3.0,<1.8.0
+dbt-bigquery>=1.3.0,<1.8.0
+dbt-redshift>=1.3.0,<1.8.0
+dbt-postgres>=1.3.0,<1.8.0
+dbt-spark>=1.3.0,<1.8.0
+dbt-spark[PyHive]>=1.3.0,<1.8.0
+dbt-databricks>=1.6.0,<1.8.0

macros/get_entity_columns.sql

@@ -3,45 +3,20 @@
 {% set columns = [
     {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "altemail", "datatype": dbt.type_string()},
-    {"name": "altname", "datatype": dbt.type_string()},
-    {"name": "altphone", "datatype": dbt.type_string()},
-    {"name": "comments", "datatype": dbt.type_string()},
     {"name": "contact", "datatype": dbt.type_int()},
-    {"name": "customer", "datatype": dbt.type_int()},
-    {"name": "date_deleted", "datatype": dbt.type_timestamp()},
-    {"name": "datecreated", "datatype": dbt.type_timestamp()},
-    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "customer", "datatype": dbt.type_int()}, 
     {"name": "employee", "datatype": dbt.type_int()},
-    {"name": "entityid", "datatype": dbt.type_string()},
-    {"name": "entitynumber", "datatype": dbt.type_int()},
     {"name": "entitytitle", "datatype": dbt.type_string()},
-    {"name": "externalid", "datatype": dbt.type_string()},
-    {"name": "fax", "datatype": dbt.type_string()},
-    {"name": "firstname", "datatype": dbt.type_string()},
-    {"name": "genericresource", "datatype": dbt.type_int()},
-    {"name": "homephone", "datatype": dbt.type_string()},
     {"name": "id", "datatype": dbt.type_int()},
-    {"name": "isinactive", "datatype": dbt.type_string()},
     {"name": "isperson", "datatype": dbt.type_string()},
-    {"name": "laborcost", "datatype": dbt.type_float()},
-    {"name": "lastmodifieddate", "datatype": dbt.type_timestamp()},
-    {"name": "lastname", "datatype": dbt.type_string()},
-    {"name": "middlename", "datatype": dbt.type_string()},
-    {"name": "mobilephone", "datatype": dbt.type_string()},
-    {"name": "othername", "datatype": dbt.type_int()},
     {"name": "parent", "datatype": dbt.type_int()},
-    {"name": "partner", "datatype": dbt.type_int()},
-    {"name": "phone", "datatype": dbt.type_string()},
     {"name": "project", "datatype": dbt.type_int()},
-    {"name": "projecttemplate", "datatype": dbt.type_int()},
-    {"name": "salutation", "datatype": dbt.type_string()},
-    {"name": "title", "datatype": dbt.type_string()},
-    {"name": "toplevelparent", "datatype": dbt.type_int()},
     {"name": "type", "datatype": dbt.type_string()},
     {"name": "vendor", "datatype": dbt.type_int()}
 ] %}
 
+{{ fivetran_utils.add_pass_through_columns(columns, var('entities_pass_through_columns')) }}
+
 {{ return(columns) }}
 
 {% endmacro %}

models/netsuite2/stg_netsuite2__entities.sql

@@ -32,6 +32,10 @@ final as (
         employee as employee_id,
         project as job_id,
         vendor as vendor_id
+
+        --The below macro adds the fields defined within your entities_pass_through_columns variable into the staging model
+        {{ fivetran_utils.fill_pass_through_columns('entities_pass_through_columns') }}
+
     from fields
 )