Merge pull request #39 from fivetran/feature/account-eliminate-field-add

feature/account-eliminate-field-add
fivetran · Oct 18, 2023 · b51f316 · b51f316
2 parents 8255c87 + cd21d7e
commit b51f316
Show file tree

Hide file tree

Showing 14 changed files with 52 additions and 27 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+# dbt_netsuite_source v0.8.0
+## 🚨 Breaking Changes 🚨:
+- Addition of the `is_eliminate` and `special_account_type_id` fields within the `stg_netsuite2__accounts` model. ([PR #39](https://github.com/fivetran/dbt_netsuite_source/pull/39) and [PR #40](https://github.com/fivetran/dbt_netsuite_source/pull/40))
+  - This addition likely won't be breaking for a large number of users. However, if you are leveraging the `accounts_pass_through_columns` variable and have included either `eliminate` or `sspecacct` as a pass through columns then you will experience a duplicate column error. We advise you remove the `eliminate` and `sspecacct` fields as a pass through columns to avoid runtime errors following this upgrade.
+
+## Contributors:
+- [@jmongerlyra](https://github.com/jmongerlyra) ([PR #40](https://github.com/fivetran/dbt_netsuite_source/pull/40))
+
 # dbt_netsuite_source v0.7.0
 ## 🚨 Breaking Changes 🚨:
 

diff --git a/README.md b/README.md
@@ -89,7 +89,7 @@ If you  are **not** using the [Netsuite transformation package](https://github.c
 ```yaml
 packages:
   - package: fivetran/netsuite_source
-    version: [">=0.7.0", "<0.8.0"]
+    version: [">=0.8.0", "<0.9.0"]
 ```
 
 ## Step 3: Define Netsuite.com or Netsuite2 Source

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.7.0'
+version: '0.8.0'
 
 models:
   netsuite_source:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,6 +1,6 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.7.0'
+version: '0.8.0'
 
 profile: 'integration_tests'
 config-version: 2
@@ -54,6 +54,9 @@ seeds:
     +column_types:
       ending: timestamp
       due_date: timestamp
+    netsuite2_account_data:
+      +column_types:
+        eliminate: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -4,4 +4,6 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+
+oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437
diff --git a/integration_tests/seeds/netsuite2_account_data.csv b/integration_tests/seeds/netsuite2_account_data.csv
@@ -1,7 +1,7 @@
-id,_fivetran_deleted,_fivetran_synced,accountsearchdisplayname,accountsearchdisplaynamecopy,acctnumber,accttype,billableexpensesacct,cashflowrate,category1099misc,class,currency,custrecord_fam_account_showinfixedasset,date_deleted,deferralacct,department,description,displaynamewithhierarchy,eliminate,externalid,fullname,generalrate,includechildren,inventory,isinactive,issummary,lastmodifieddate,location,parent,reconcilewithmatching,revalue,sbankname,sbankroutingnumber,sspecacct,subsidiary
-133,FALSE,2022-04-07 7:05:11,Power,Power,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Power,,Allowance for Doubtful Account,Power,AVERAGE,T,F,F,F,2017-05-21 1:41:07,,,F,F,,,Power,"1, 3, 4, 6"
-157,FALSE,2022-04-07 7:05:11,Levels,Levels,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Levels,,,Levels,AVERAGE,T,F,F,F,2017-05-02 22:46:13,,,F,F,,,Levels,"1, 3, 4, 6"
-118,FALSE,2022-04-07 7:05:11,Are,Are,,NonPosting,,AVERAGE,,,,,,,,,Are,,,Are,AVERAGE,T,F,F,F,,,,F,F,,,Are,"1, 3, 4, 6"
-125,FALSE,2022-04-07 7:05:11,Over,Over,,NonPosting,,AVERAGE,,,,,,,,,Over,,,Over,AVERAGE,T,F,F,F,,,,F,F,,,Over,"1, 3, 4, 6"
-121,FALSE,2022-04-07 7:05:11,Nine,Nine,,COGS,,AVERAGE,,,,,,,,,Nine,,,Nine,AVERAGE,T,F,F,F,2021-02-10 9:19:54,,,F,F,,,Nine,"1, 3, 4, 6"
-117,FALSE,2022-04-07 7:05:11,Thousand,Thousand,,NonPosting,,AVERAGE,,,,,,,,,Thousand,,,Thousand,AVERAGE,T,F,F,F,,,,F,F,,,Thousand,"1, 3, 4, 6"
+id,_fivetran_deleted,_fivetran_synced,accountsearchdisplayname,accountsearchdisplaynamecopy,acctnumber,accttype,billableexpensesacct,cashflowrate,category1099misc,class,currency,custrecord_fam_account_showinfixedasset,date_deleted,deferralacct,department,description,displaynamewithhierarchy,eliminate,externalid,fullname,generalrate,includechildren,inventory,isinactive,issummary,lastmodifieddate,location,parent,reconcilewithmatching,revalue,sbankname,sbankroutingnumber,sspecacct,subsidiary,eliminate,sspecacct
+133,FALSE,2022-04-07 7:05:11,Power,Power,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Power,,Allowance for Doubtful Account,Power,AVERAGE,T,F,F,F,2017-05-21 1:41:07,,,F,F,,,Power,"1, 3, 4, 6",F,Tax
+157,FALSE,2022-04-07 7:05:11,Levels,Levels,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Levels,,,Levels,AVERAGE,T,F,F,F,2017-05-02 22:46:13,,,F,F,,,Levels,"1, 3, 4, 6",F,
+118,FALSE,2022-04-07 7:05:11,Are,Are,,NonPosting,,AVERAGE,,,,,,,,,Are,,,Are,AVERAGE,T,F,F,F,,,,F,F,,,Are,"1, 3, 4, 6",F,
+125,FALSE,2022-04-07 7:05:11,Over,Over,,NonPosting,,AVERAGE,,,,,,,,,Over,,,Over,AVERAGE,T,F,F,F,,,,F,F,,,Over,"1, 3, 4, 6",F,CTA-E
+121,FALSE,2022-04-07 7:05:11,Nine,Nine,,COGS,,AVERAGE,,,,,,,,,Nine,,,Nine,AVERAGE,T,F,F,F,2021-02-10 9:19:54,,,F,F,,,Nine,"1, 3, 4, 6",,
+117,FALSE,2022-04-07 7:05:11,Thousand,Thousand,,NonPosting,,AVERAGE,,,,,,,,,Thousand,,,Thousand,AVERAGE,T,F,F,F,,,,F,F,,,Thousand,"1, 3, 4, 6",T,
diff --git a/macros/get_accounts_columns.sql b/macros/get_accounts_columns.sql
@@ -29,6 +29,7 @@
     {"name": "parent", "datatype": dbt.type_int()},
     {"name": "acctnumber", "datatype": dbt.type_string()},
     {"name": "accttype", "datatype": dbt.type_string()},
+    {"name": "sspecacct", "datatype": dbt.type_string()},
     {"name": "fullname", "datatype": dbt.type_string()},
     {"name": "description", "datatype": dbt.type_string()},
     {"name": "deferralacct", "datatype": dbt.type_int()},
@@ -40,7 +41,8 @@
     {"name": "location", "datatype": dbt.type_int()},
     {"name": "includechildren", "datatype": dbt.type_string()},
     {"name": "isinactive", "datatype": dbt.type_string()},
-    {"name": "issummary", "datatype": dbt.type_string()}
+    {"name": "issummary", "datatype": dbt.type_string()},
+    {"name": "eliminate", "datatype": dbt.type_string()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('accounts_pass_through_columns')) }}

diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml
@@ -142,6 +142,8 @@ sources:
             description: Netsuite generated account number.
           - name: accttype
             description: Reference to the account type.
+          - name: sspecacct
+            description: Special account type.
           - name: fullname
             description: Name of the account.
           - name: description
@@ -166,6 +168,8 @@ sources:
             description: Boolean indicating if the account is inactive. Represented as "T" or "F" for true and false respectively.
           - name: issummary
             description: Boolean indicating if the account is a summary account. Represented as "T" or "F" for true and false respectively.
+          - name: eliminate
+            description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. Represented as "T" or "F" for true and false respectively.
           - name: _fivetran_deleted
             description: "{{ doc('_fivetran_deleted') }}"
 

diff --git a/models/netsuite2/stg_netsuite2__accounts.sql b/models/netsuite2/stg_netsuite2__accounts.sql
@@ -27,6 +27,7 @@ final as (
         parent as parent_id,
         acctnumber as account_number,
         accttype as account_type_id,
+        sspecacct as special_account_type_id,
         fullname as name,
         description as account_description,
         deferralacct as deferral_account_id,
@@ -39,6 +40,7 @@ final as (
         includechildren = 'T' as is_including_child_subs,
         isinactive = 'T' as is_inactive,
         issummary = 'T' as is_summary,
+        eliminate = 'T' as is_eliminate,
         _fivetran_deleted
 
         --The below macro adds the fields defined within your accounts_pass_through_columns variable into the staging model

diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml
@@ -136,6 +136,8 @@ models:
         description: Netsuite generate account number.
       - name: account_type_id
         description: Reference to the account type.
+      - name: special_account_type_id
+        description: Special account type.
       - name: name
         description: Name of the account.
       - name: account_description
@@ -160,6 +162,8 @@ models:
         description: Boolean indicating if the account is inactive.
       - name: is_summary
         description: Boolean indicating if the account is a summary account.
+      - name: is_eliminate
+        description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period.
       - name: _fivetran_deleted
         description: "{{ doc('_fivetran_deleted') }}"