From daaa5a29274b7fa97628d81413ac915926a8ff76 Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:17:45 -0500 Subject: [PATCH 01/16] Add lyra_subledger --- dbt_project.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 952bb12..1b792fd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -61,7 +61,11 @@ vars: alias: "special_account_type_id" classes_pass_through_columns: [] departments_pass_through_columns: [] - transactions_pass_through_columns: [] + transactions_pass_through_columns: + - name: "custbody_subledger" + alias: "lyra_subledger" + - name: "custbody_subledger_id" + alias: "lyra_subledger_id" transaction_lines_pass_through_columns: [] customers_pass_through_columns: [] locations_pass_through_columns: [] From 3eb3866b50baebc83ec198cfcb745e1de158b93c Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:54:34 -0600 Subject: [PATCH 02/16] Add fields to transactions and transaction_lines (#7) Add extra fields for intercompany reconciliation. --- macros/get_subsidiaries_columns.sql | 3 ++- macros/get_transaction_lines_columns.sql | 4 +++- models/netsuite2/src_netsuite2.yml | 6 ++++++ models/netsuite2/stg_netsuite2__subsidiaries.sql | 3 ++- .../stg_netsuite2__transaction_accounting_lines.sql | 1 + models/netsuite2/stg_netsuite2__transaction_lines.sql | 4 +++- models/stg_netsuite2.yml | 6 ++++++ 7 files changed, 23 insertions(+), 4 deletions(-) diff --git a/macros/get_subsidiaries_columns.sql b/macros/get_subsidiaries_columns.sql index b1e5efb..286d3fb 100644 --- a/macros/get_subsidiaries_columns.sql +++ b/macros/get_subsidiaries_columns.sql @@ -76,7 +76,8 @@ {"name": "country", "datatype": dbt.type_string()}, {"name": "state", "datatype": dbt.type_string()}, {"name": "fiscalcalendar", "datatype": dbt.type_int()}, - {"name": "parent", "datatype": dbt.type_int()} + {"name": "parent", "datatype": dbt.type_int()}, + {"name": "iselimination", "datatype": dbt.type_string()} ] %} {{ fivetran_utils.add_pass_through_columns(columns, var('subsidiaries_pass_through_columns')) }} diff --git a/macros/get_transaction_lines_columns.sql b/macros/get_transaction_lines_columns.sql index a2cab97..b1629d7 100644 --- a/macros/get_transaction_lines_columns.sql +++ b/macros/get_transaction_lines_columns.sql @@ -42,7 +42,9 @@ {"name": "cleared", "datatype": dbt.type_string()}, {"name": "commitmentfirm", "datatype": dbt.type_string()}, {"name": "mainline", "datatype": dbt.type_string()}, - {"name": "taxline", "datatype": dbt.type_string()} + {"name": "taxline", "datatype": dbt.type_string()}, + {"name": "eliminate", "datatype": dbt.type_string()}, + {"name": "netamount", "datatype": dbt.type_float()} ] %} {{ fivetran_utils.add_pass_through_columns(columns, var('transaction_lines_pass_through_columns')) }} diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml index 386771e..c4c8f36 100644 --- a/models/netsuite2/src_netsuite2.yml +++ b/models/netsuite2/src_netsuite2.yml @@ -470,6 +470,8 @@ sources: description: "{{ doc('fiscal_calendar_id') }}" - name: parent description: Reference to the parent subsidiary. + - name: iselimination + description: Boolean indicating if the subsidiary is an elimination subsidiary. Represented as "T" or "F" for true and false respectively. - name: transaction_accounting_line identifier: "{{ var('netsuite2_transaction_accounting_line_identifier', 'transactionaccountingline') }}" @@ -538,6 +540,10 @@ sources: description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively. - name: taxline description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. + - name: eliminate + description: Boolean indicating if the transaction line is eliminated. Represented as "T" or "F" for true and false respectively. + - name: netamount + description: Net amount of transaction line. - name: transaction identifier: "{{ var('netsuite2_transaction_identifier', 'transaction') }}" diff --git a/models/netsuite2/stg_netsuite2__subsidiaries.sql b/models/netsuite2/stg_netsuite2__subsidiaries.sql index fcac161..2f21575 100644 --- a/models/netsuite2/stg_netsuite2__subsidiaries.sql +++ b/models/netsuite2/stg_netsuite2__subsidiaries.sql @@ -30,7 +30,8 @@ final as ( country, state, fiscalcalendar as fiscal_calendar_id, - parent as parent_id + parent as parent_id, + iselimination = 'T' as is_elimination --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('subsidiaries_pass_through_columns') }} diff --git a/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql b/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql index 2a6dfb0..2eb5fc8 100644 --- a/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql +++ b/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql @@ -28,6 +28,7 @@ final as ( accountingbook as accounting_book_id, account as account_id, posting = 'T' as is_posting, + exchangerate as exchange_rate, amount, credit as credit_amount, debit as debit_amount, diff --git a/models/netsuite2/stg_netsuite2__transaction_lines.sql b/models/netsuite2/stg_netsuite2__transaction_lines.sql index a6f79b4..ba24b12 100644 --- a/models/netsuite2/stg_netsuite2__transaction_lines.sql +++ b/models/netsuite2/stg_netsuite2__transaction_lines.sql @@ -38,7 +38,9 @@ final as ( cleared = 'T' as is_cleared, commitmentfirm = 'T' as is_commitment_firm, mainline = 'T' as is_main_line, - taxline = 'T' as is_tax_line + taxline = 'T' as is_tax_line, + eliminate = 'T' as is_eliminate, + netamount --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }} diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml index b05f592..1b060d9 100644 --- a/models/stg_netsuite2.yml +++ b/models/stg_netsuite2.yml @@ -482,6 +482,8 @@ models: description: "{{ doc('fiscal_calendar_id') }}" - name: parent_id description: Reference to the parent subsidiary. + - name: is_elimination + description: Boolean indicating if the subsidiary is an elimination subsidiary. - name: stg_netsuite2__transaction_accounting_lines description: "{{ doc('transaction_accounting_lines_table') }}" @@ -570,6 +572,10 @@ models: description: Boolean indicating if the transaction line is a main line entry. - name: is_tax_line description: Boolean indicating if the transaction line is a tax line. + - name: is_eliminate + description: Boolean indicating if the transaction line is eliminated. + - name: netamount + description: Net amount of transaction line. - name: stg_netsuite2__transactions description: "{{ doc('transaction_table') }}" From 5b3500968d50baac6829ad90868af0a3853b679f Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:06:45 -0600 Subject: [PATCH 03/16] Move special_account_type_id and currency_id to staging tables (#8) * Align sspecacct with upstream * Add currency_id to stg_netsuite2__subsidiaries * Fix currency_id * Update get_subsidiaries_columns.sql * Add type_based_document_number --- dbt_project.yml | 8 ++++---- macros/get_subsidiaries_columns.sql | 3 ++- models/netsuite2/stg_netsuite2__accounts.sql | 1 + models/netsuite2/stg_netsuite2__subsidiaries.sql | 3 ++- models/stg_netsuite2.yml | 2 ++ 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 13093be..9d30eef 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -56,9 +56,7 @@ vars: netsuite2_account_types: "{{ source('netsuite2', 'account_type') }}" netsuite2_entity_address: "{{ source('netsuite2', 'entity_address') }}" netsuite2_location_main_address: "{{ source('netsuite2', 'location_main_address') }}" - accounts_pass_through_columns: - - name: "sspecacct" - alias: "special_account_type_id" + accounts_pass_through_columns: [] classes_pass_through_columns: [] departments_pass_through_columns: [] transactions_pass_through_columns: @@ -66,10 +64,12 @@ vars: alias: "lyra_subledger" - name: "custbody_subledger_id" alias: "lyra_subledger_id" + - name: "typebaseddocumentnumber" + alias: "type_based_document_number" transaction_lines_pass_through_columns: [] customers_pass_through_columns: [] locations_pass_through_columns: [] - subsidiaries_pass_through_columns: [currency] + subsidiaries_pass_through_columns: [] consolidated_exchange_rates_pass_through_columns: [] vendors_pass_through_columns: [] items_pass_through_columns: [] diff --git a/macros/get_subsidiaries_columns.sql b/macros/get_subsidiaries_columns.sql index 286d3fb..e77ad1e 100644 --- a/macros/get_subsidiaries_columns.sql +++ b/macros/get_subsidiaries_columns.sql @@ -77,7 +77,8 @@ {"name": "state", "datatype": dbt.type_string()}, {"name": "fiscalcalendar", "datatype": dbt.type_int()}, {"name": "parent", "datatype": dbt.type_int()}, - {"name": "iselimination", "datatype": dbt.type_string()} + {"name": "iselimination", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_int()} ] %} {{ fivetran_utils.add_pass_through_columns(columns, var('subsidiaries_pass_through_columns')) }} diff --git a/models/netsuite2/stg_netsuite2__accounts.sql b/models/netsuite2/stg_netsuite2__accounts.sql index 2340edb..f55b702 100644 --- a/models/netsuite2/stg_netsuite2__accounts.sql +++ b/models/netsuite2/stg_netsuite2__accounts.sql @@ -27,6 +27,7 @@ final as ( parent as parent_id, acctnumber as account_number, accttype as account_type_id, + sspecacct as special_account_type_id, fullname as name, description as account_description, deferralacct as deferral_account_id, diff --git a/models/netsuite2/stg_netsuite2__subsidiaries.sql b/models/netsuite2/stg_netsuite2__subsidiaries.sql index 2f21575..50bb280 100644 --- a/models/netsuite2/stg_netsuite2__subsidiaries.sql +++ b/models/netsuite2/stg_netsuite2__subsidiaries.sql @@ -31,7 +31,8 @@ final as ( state, fiscalcalendar as fiscal_calendar_id, parent as parent_id, - iselimination = 'T' as is_elimination + iselimination = 'T' as is_elimination, + currency as currency_id --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('subsidiaries_pass_through_columns') }} diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml index 1b060d9..ccfdc0d 100644 --- a/models/stg_netsuite2.yml +++ b/models/stg_netsuite2.yml @@ -484,6 +484,8 @@ models: description: Reference to the parent subsidiary. - name: is_elimination description: Boolean indicating if the subsidiary is an elimination subsidiary. + - name: currency_id + description: "{{ doc('currency_id') }}" - name: stg_netsuite2__transaction_accounting_lines description: "{{ doc('transaction_accounting_lines_table') }}" From 93faee865ffd400150beed6f66e83d7e57626af5 Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:58:00 -0600 Subject: [PATCH 04/16] Add account display_name (#9) --- macros/get_accounts_columns.sql | 1 + models/netsuite2/stg_netsuite2__accounts.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/macros/get_accounts_columns.sql b/macros/get_accounts_columns.sql index 1771adb..66fc972 100644 --- a/macros/get_accounts_columns.sql +++ b/macros/get_accounts_columns.sql @@ -31,6 +31,7 @@ {"name": "accttype", "datatype": dbt.type_string()}, {"name": "sspecacct", "datatype": dbt.type_string()}, {"name": "fullname", "datatype": dbt.type_string()}, + {"name": "accountsearchdisplaynamecopy", "datatype": dbt.type_string()}, {"name": "description", "datatype": dbt.type_string()}, {"name": "deferralacct", "datatype": dbt.type_int()}, {"name": "cashflowrate", "datatype": dbt.type_string()}, diff --git a/models/netsuite2/stg_netsuite2__accounts.sql b/models/netsuite2/stg_netsuite2__accounts.sql index f55b702..f4bd496 100644 --- a/models/netsuite2/stg_netsuite2__accounts.sql +++ b/models/netsuite2/stg_netsuite2__accounts.sql @@ -29,6 +29,7 @@ final as ( accttype as account_type_id, sspecacct as special_account_type_id, fullname as name, + accountsearchdisplaynamecopy as display_name, description as account_description, deferralacct as deferral_account_id, cashflowrate as cash_flow_rate_type, From 5e79409f4743fa1b00fe23ae03b93aa6f98f6958 Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:26:52 -0600 Subject: [PATCH 05/16] Add lyra_cash_flow_category (#10) --- dbt_project.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 9d30eef..31f9310 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -56,7 +56,9 @@ vars: netsuite2_account_types: "{{ source('netsuite2', 'account_type') }}" netsuite2_entity_address: "{{ source('netsuite2', 'entity_address') }}" netsuite2_location_main_address: "{{ source('netsuite2', 'location_main_address') }}" - accounts_pass_through_columns: [] + accounts_pass_through_columns: + - name: "custrecord_cash_flow_category" + alias: "lyra_cash_flow_category" classes_pass_through_columns: [] departments_pass_through_columns: [] transactions_pass_through_columns: From 0351cfc9c44d46f59ee4ddeb3567b52c1e3a3b40 Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:21:09 -0600 Subject: [PATCH 06/16] Add created_date and last_modified_date (#11) --- dbt_project.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 31f9310..4785bf7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -68,6 +68,10 @@ vars: alias: "lyra_subledger_id" - name: "typebaseddocumentnumber" alias: "type_based_document_number" + - name: "createddate" + alias: "created_date" + - name: "lastmodifieddate" + alias: "last_modified_date" transaction_lines_pass_through_columns: [] customers_pass_through_columns: [] locations_pass_through_columns: [] From 1ac84558af6b836c727c1f8d7338137bd35256df Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:58:33 -0500 Subject: [PATCH 07/16] Add alt_name (#13) --- macros/get_customers_columns.sql | 1 + models/docs.md | 2 ++ models/netsuite2/stg_netsuite2__customers.sql | 1 + models/netsuite2/stg_netsuite2__vendors.sql | 1 + models/stg_netsuite2.yml | 4 ++++ 5 files changed, 9 insertions(+) diff --git a/macros/get_customers_columns.sql b/macros/get_customers_columns.sql index a8f2542..7a6a9f5 100644 --- a/macros/get_customers_columns.sql +++ b/macros/get_customers_columns.sql @@ -27,6 +27,7 @@ {"name": "externalid", "datatype": dbt.type_string()}, {"name": "parent", "datatype": dbt.type_int()}, {"name": "isperson", "datatype": dbt.type_string()}, + {"name": "altname", "datatype": dbt.type_string()}, {"name": "companyname", "datatype": dbt.type_string()}, {"name": "firstname", "datatype": dbt.type_string()}, {"name": "lastname", "datatype": dbt.type_string()}, diff --git a/models/docs.md b/models/docs.md index b18b185..3fdf5b2 100644 --- a/models/docs.md +++ b/models/docs.md @@ -61,6 +61,8 @@ {% docs vendor_id %} The unique identifier of the vendor. {% enddocs %} +{% docs alt_name %} Name of the person or company. {% enddocs %} + {% docs company_name %} Name of the company. {% enddocs %} {% docs create_date_at %} Timestamp of the record creation. {% enddocs %} diff --git a/models/netsuite2/stg_netsuite2__customers.sql b/models/netsuite2/stg_netsuite2__customers.sql index 8a9fca1..cc3303e 100644 --- a/models/netsuite2/stg_netsuite2__customers.sql +++ b/models/netsuite2/stg_netsuite2__customers.sql @@ -26,6 +26,7 @@ final as ( externalid as customer_external_id, parent as parent_id, isperson = 'T' as is_person, + altname as alt_name, companyname as company_name, firstname as first_name, lastname as last_name, diff --git a/models/netsuite2/stg_netsuite2__vendors.sql b/models/netsuite2/stg_netsuite2__vendors.sql index 22a26bb..a5c2d78 100644 --- a/models/netsuite2/stg_netsuite2__vendors.sql +++ b/models/netsuite2/stg_netsuite2__vendors.sql @@ -24,6 +24,7 @@ final as ( select _fivetran_synced, id as vendor_id, + altname as alt_name, companyname as company_name, datecreated as create_date_at, category as vendor_category_id diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml index 9696ead..fd9139f 100644 --- a/models/stg_netsuite2.yml +++ b/models/stg_netsuite2.yml @@ -246,6 +246,8 @@ models: description: Reference to the parent customer. - name: is_person description: Boolean indicating if the customer is an individual person. + - name: alt_name + description: "{{ doc('alt_name') }}" - name: company_name description: "{{ doc('company_name') }}" - name: first_name @@ -642,6 +644,8 @@ models: tests: - unique - not_null + - name: alt_name + description: "{{ doc('alt_name') }}" - name: company_name description: "{{ doc('company_name') }}" - name: create_date_at From 74a101f5588a775eb6ef882043297ffea63d2ece Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:45:15 -0500 Subject: [PATCH 08/16] Add lyra_sec_opex_category (#14) --- dbt_project.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index b25a7c1..eaa87f9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -60,7 +60,9 @@ vars: - name: "custrecord_cash_flow_category" alias: "lyra_cash_flow_category" classes_pass_through_columns: [] - departments_pass_through_columns: [] + departments_pass_through_columns: + - name: "custrecord_sec_opex_category" + alias: "lyra_sec_opex_category" transactions_pass_through_columns: - name: "custbody_subledger" alias: "lyra_subledger" From 5048c57b7c7c52adff3a3528e5d91dfc95ce7f3d Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Wed, 10 Apr 2024 07:13:48 -0500 Subject: [PATCH 09/16] Add stg_netsuite2__employees & lyra_ebitda_category (#15) --- dbt_project.yml | 7 +++ macros/get_employee_columns.sql | 35 +++++++++++++++ macros/get_transactions_columns.sql | 2 + models/docs.md | 6 +++ models/netsuite2/src_netsuite2.yml | 35 +++++++++++++++ models/netsuite2/stg_netsuite2__employees.sql | 43 +++++++++++++++++++ .../tmp/stg_netsuite2__employees_tmp.sql | 4 ++ models/stg_netsuite2.yml | 37 ++++++++++++++++ 8 files changed, 169 insertions(+) create mode 100644 macros/get_employee_columns.sql create mode 100644 models/netsuite2/stg_netsuite2__employees.sql create mode 100644 models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql diff --git a/dbt_project.yml b/dbt_project.yml index eaa87f9..fd1d2d0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -41,6 +41,7 @@ vars: netsuite2_currencies: "{{ source('netsuite2','currency') }}" netsuite2_customers: "{{ source('netsuite2','customer') }}" netsuite2_departments: "{{ source('netsuite2','department') }}" + netsuite2_employees: "{{ source('netsuite2','employee') }}" netsuite2_entities: "{{ source('netsuite2','entity') }}" netsuite2_items: "{{ source('netsuite2','item') }}" netsuite2_jobs: "{{ source('netsuite2','job') }}" @@ -59,6 +60,8 @@ vars: accounts_pass_through_columns: - name: "custrecord_cash_flow_category" alias: "lyra_cash_flow_category" + - name: "custrecord_ebitda_category" + alias: "lyra_ebitda_category" classes_pass_through_columns: [] departments_pass_through_columns: - name: "custrecord_sec_opex_category" @@ -70,8 +73,12 @@ vars: alias: "lyra_subledger_id" - name: "typebaseddocumentnumber" alias: "type_based_document_number" + - name: "createdby" + alias: "created_by_id" - name: "createddate" alias: "created_date" + - name: "lastmodifiedby" + alias: "last_modified_by_id" - name: "lastmodifieddate" alias: "last_modified_date" transaction_lines_pass_through_columns: [] diff --git a/macros/get_employee_columns.sql b/macros/get_employee_columns.sql new file mode 100644 index 0000000..de41b6f --- /dev/null +++ b/macros/get_employee_columns.sql @@ -0,0 +1,35 @@ +{% macro get_employee_columns() %} + +{% set columns = [ + {"name": "_fivetran_deleted", "datatype": "boolean"}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "accountnumber", "datatype": dbt.type_string()}, + {"name": "aliennumber", "datatype": dbt.type_string()}, + {"name": "approvallimit", "datatype": dbt.type_float()}, + {"name": "approver", "datatype": dbt.type_int()}, + {"name": "class", "datatype": dbt.type_int()}, + {"name": "comments", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_int()}, + {"name": "department", "datatype": dbt.type_int()}, + {"name": "email", "datatype": dbt.type_string()}, + {"name": "employeestatus", "datatype": dbt.type_int()}, + {"name": "employeetype", "datatype": dbt.type_int()}, + {"name": "entityid", "datatype": dbt.type_string()}, + {"name": "expenselimit", "datatype": dbt.type_string()}, + {"name": "firstname", "datatype": dbt.type_string()}, + {"name": "giveaccess", "datatype": dbt.type_string()}, + {"name": "hiredate", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_int()}, + {"name": "isinactive", "datatype": dbt.type_string()}, + {"name": "lastname", "datatype": dbt.type_string()}, + {"name": "location", "datatype": dbt.type_int()}, + {"name": "middlename", "datatype": dbt.type_string()}, + {"name": "purchaseorderapprovallimit", "datatype": dbt.type_float()}, + {"name": "purchaseorderlimit", "datatype": dbt.type_float()}, + {"name": "subsidiary", "datatype": dbt.type_int()}, + {"name": "supervisor", "datatype": dbt.type_int()} +] %} + +{{ return(columns) }} + +{% endmacro %} diff --git a/macros/get_transactions_columns.sql b/macros/get_transactions_columns.sql index c3d180a..9ddf5a4 100644 --- a/macros/get_transactions_columns.sql +++ b/macros/get_transactions_columns.sql @@ -29,11 +29,13 @@ {"name": "memo", "datatype": dbt.type_string()}, {"name": "trandate", "datatype": dbt.type_timestamp()}, {"name": "status", "datatype": dbt.type_string()}, + {"name": "createdby", "datatype": dbt.type_int()}, {"name": "createddate", "datatype": dbt.type_timestamp()}, {"name": "duedate", "datatype": dbt.type_timestamp()}, {"name": "closedate", "datatype": dbt.type_timestamp()}, {"name": "currency", "datatype": dbt.type_int()}, {"name": "entity", "datatype": dbt.type_int()}, + {"name": "lastmodifiedby", "datatype": dbt.type_int()}, {"name": "postingperiod", "datatype": dbt.type_int()}, {"name": "posting", "datatype": dbt.type_string()}, {"name": "intercoadj", "datatype": dbt.type_string()}, diff --git a/models/docs.md b/models/docs.md index 3fdf5b2..33d641d 100644 --- a/models/docs.md +++ b/models/docs.md @@ -36,6 +36,8 @@ {% docs entities_table %} Table detailing all entities in Netsuite. {% enddocs %} +{% docs employee_table %} Table detailing all employees in Netsuite. {% enddocs %} + {% docs jobs_table %} Table detailing all jobs. {% enddocs %} {% docs transaction_accounting_lines_table %} A table detailing all transaction lines for all transactions. {% enddocs %} @@ -77,6 +79,10 @@ {% docs entity_id %} The entity id of the entity used for the record. {% enddocs %} +{% docs employee_id %} The employee id of the employee used for the record. {% enddocs %} + +{% docs supervisor_id %} The employee id of the employee's supervisor used for the record. {% enddocs %} + {% docs transaction_id %} The transaction id of referenced for the record. {% enddocs %} {% docs department_id %} The unique identifier of the department used for the record. {% enddocs %} diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml index 624bf18..6aeb958 100644 --- a/models/netsuite2/src_netsuite2.yml +++ b/models/netsuite2/src_netsuite2.yml @@ -286,6 +286,41 @@ sources: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: employee + identified: "{{ var('netsuite2_employee_identifier', 'employee') }}" + description: "{{ doc('employee_table') }}" + columns: + - name: _fivetran_synced + description: "{{ doc('_fivetran_synced') }}" + - name: id + description: "{{ doc('employee_id') }}" + - name: entityid + description: "{{ doc('entity_id') }}" + - name: firstname + description: First name of the employee. + - name: lastname + description: Last name of the employee. + - name: department + description: "{{ doc('department_id') }}" + - name: subsidiary + description: "{{ doc('subsidiary_id') }}" + - name: email + description: Email address of the employee. + - name: supervisor + description: "{{ doc('supervisor_id') }}" + - name: approvallimit + description: The largest expense report this supervisor can approve. + - name: expenselimit + description: Amount this employee can expense without approval from a supervisor or approver. + - name: purchaseorderapprovallimit + description: Amount an employee is allowed to approve on a purchase request when specified as an approver. + - name: purchaseorderlimit + description: Amount this employee can purchase without approval from a supervisor or approver. + - name: currency + description: "{{ doc('currency_id') }}" + - name: isinactive + description: Boolean indicating this employee is active. + - name: entity identifier: "{{ var('netsuite2_entity_identifier', 'entity') }}" description: "{{ doc('entities_table') }}" diff --git a/models/netsuite2/stg_netsuite2__employees.sql b/models/netsuite2/stg_netsuite2__employees.sql new file mode 100644 index 0000000..b5a0d61 --- /dev/null +++ b/models/netsuite2/stg_netsuite2__employees.sql @@ -0,0 +1,43 @@ +{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} + +with base as ( + + select * + from {{ ref('stg_netsuite2__employees_tmp') }} +), + +fields as ( + + select + {{ + fivetran_utils.fill_staging_columns( + source_columns=adapter.get_columns_in_relation(ref('stg_netsuite2__employees_tmp')), + staging_columns=get_employee_columns() + ) + }} + from base +), + +final as ( + + select + _fivetran_synced, + id as employee_id, + entityid as entity_id, + firstname as first_name, + lastname as last_name, + department as department_id, + subsidiary as subsidiary_id, + email, + supervisor as supervisor_id, + approvallimit as approval_limit, + expenselimit as expense_limit, + purchaseorderapprovallimit as purchase_order_approval_limit, + purchaseorderlimit as purchase_order_limit, + currency as currency_id, + isinactive = 'T' as is_inactive + from fields +) + +select * +from final diff --git a/models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql new file mode 100644 index 0000000..bb16dae --- /dev/null +++ b/models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql @@ -0,0 +1,4 @@ +{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} + +select * +from {{ var('netsuite2_employees') }} diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml index fd9139f..84dc93c 100644 --- a/models/stg_netsuite2.yml +++ b/models/stg_netsuite2.yml @@ -292,6 +292,43 @@ models: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: stg_netsuite2__employees + description: "{{ doc('employee_table') }}" + columns: + - name: _fivetran_synced + description: "{{ doc('_fivetran_synced') }}" + - name: employee_id + description: "{{ doc('employee_id') }}" + tests: + - unique + - not_null + - name: entity_id + description: "{{ doc('entity_id') }}" + - name: first_name + description: First name of the employee. + - name: last_name + description: Last name of the employee. + - name: department_id + description: "{{ doc('department_id') }}" + - name: subsidiary_id + description: "{{ doc('subsidiary_id') }}" + - name: email + description: Email address of the employee. + - name: supervisor_id + description: "{{ doc('supervisor_id') }}" + - name: approval_limit + description: The largest expense report this supervisor can approve. + - name: expense_limit + description: Amount this employee can expense without approval from a supervisor or approver. + - name: purchase_order_approval_limit + description: Amount an employee is allowed to approve on a purchase request when specified as an approver. + - name: purchase_order_limit + description: Amount this employee can purchase without approval from a supervisor or approver. + - name: currency_id + description: "{{ doc('currency_id') }}" + - name: is_inactive + description: Boolean indicating this employee is active. + - name: stg_netsuite2__entities description: "{{ doc('entities_table') }}" columns: From e59baf63666be54fa3ca33fd13cd8b5ba31c482b Mon Sep 17 00:00:00 2001 From: Randy Wang <124346025+rwang-lyra@users.noreply.github.com> Date: Mon, 6 May 2024 14:00:36 -0700 Subject: [PATCH 10/16] Delete .github/workflows/auto-release.yml addressing issues from "Merge upstream/v0.13.0" #35 --- .github/workflows/auto-release.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/workflows/auto-release.yml diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml deleted file mode 100644 index 8ed5853..0000000 --- a/.github/workflows/auto-release.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: 'auto release' -on: - pull_request: - types: - - closed - branches: - - main - -jobs: - call-workflow-passing-data: - if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main - secrets: inherit \ No newline at end of file From 20dd79cf29459dba523f6e25bc8462100d407cac Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:04:52 -0500 Subject: [PATCH 11/16] Add pass through columns --- dbt_project.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index c42e50c..bb76caf 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -62,6 +62,8 @@ vars: alias: "lyra_cash_flow_category" - name: "custrecord_ebitda_category" alias: "lyra_ebitda_category" + - name: "custrecord_transfer_pricing_method" + alias: "lyra_transfer_pricing_method" classes_pass_through_columns: [] departments_pass_through_columns: - name: "custrecord_sec_opex_category" @@ -81,10 +83,20 @@ vars: alias: "last_modified_by_id" - name: "lastmodifieddate" alias: "last_modified_date" - transaction_lines_pass_through_columns: [] + transaction_lines_pass_through_columns: + - name: "amortizstartdate" + alias: "amortization_start_date" + - name: "amortizationenddate" + alias: "amortization_end_date" + - name: "custcol_inv_start_date" + alias: "lyra_inv_start_date" + - name: "custcol_inv_end_date" + alias: "lyra_inv_end_date" customers_pass_through_columns: [] locations_pass_through_columns: [] - subsidiaries_pass_through_columns: [] + subsidiaries_pass_through_columns: + - name: "custrecord_transfer_pricing_markup" + alias: "lyra_transfer_pricing_markup" consolidated_exchange_rates_pass_through_columns: [] vendors_pass_through_columns: [] items_pass_through_columns: [] From 5ea62e7147f46771791bada26944b6987f2f044c Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Mon, 24 Jun 2024 14:45:12 -0500 Subject: [PATCH 12/16] Add reversal fields (#19) --- dbt_project.yml | 2 ++ macros/get_transactions_columns.sql | 3 +++ models/netsuite2/stg_netsuite2__transactions.sql | 5 ++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index bb76caf..90cd420 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -75,6 +75,8 @@ vars: alias: "lyra_subledger_id" - name: "typebaseddocumentnumber" alias: "type_based_document_number" + - name: "tranid" + alias: "tran_id" - name: "createdby" alias: "created_by_id" - name: "createddate" diff --git a/macros/get_transactions_columns.sql b/macros/get_transactions_columns.sql index 9ddf5a4..1ace50d 100644 --- a/macros/get_transactions_columns.sql +++ b/macros/get_transactions_columns.sql @@ -40,6 +40,9 @@ {"name": "posting", "datatype": dbt.type_string()}, {"name": "intercoadj", "datatype": dbt.type_string()}, {"name": "isreversal", "datatype": dbt.type_string()}, + {"name": "reversal", "datatype": dbt.type_int()}, + {"name": "reversaldate", "datatype": dbt.type_timestamp()}, + {"name": "reversaldefer", "datatype": dbt.type_string()}, {"name": "_fivetran_deleted", "datatype": "boolean"} ] %} diff --git a/models/netsuite2/stg_netsuite2__transactions.sql b/models/netsuite2/stg_netsuite2__transactions.sql index a9384ff..bb79a07 100644 --- a/models/netsuite2/stg_netsuite2__transactions.sql +++ b/models/netsuite2/stg_netsuite2__transactions.sql @@ -37,7 +37,10 @@ final as ( postingperiod as accounting_period_id, posting = 'T' as is_posting, intercoadj = 'T' as is_intercompany_adjustment, - isreversal = 'T' as is_reversal + isreversal = 'T' as is_reversal, + reversal as reversal_transaction_id, + reversaldate as reversal_date, + reversaldefer = 'T' as is_reversal_defer --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('transactions_pass_through_columns') }} From f22cf901c8115fb3fe2351d8f6894a8108509745 Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:09:32 -0500 Subject: [PATCH 13/16] Add custrecord_transfer_pricing_coa_method (#20) --- dbt_project.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 90cd420..1e01d75 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -99,6 +99,8 @@ vars: subsidiaries_pass_through_columns: - name: "custrecord_transfer_pricing_markup" alias: "lyra_transfer_pricing_markup" + - name: "custrecord_transfer_pricing_coa_method" + alias: "lyra_transfer_pricing_coa_method" consolidated_exchange_rates_pass_through_columns: [] vendors_pass_through_columns: [] items_pass_through_columns: [] From 621f4e1b68f12aae5c1d0e6eb4bc34f928f6de7c Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:50:30 -0500 Subject: [PATCH 14/16] Cast timestamp fields as date (#21) For timestamp fields not converted by the NetSuite UI, cast as date --- dbt_project.yml | 4 ++++ models/netsuite2/stg_netsuite2__accounting_periods.sql | 4 ++-- models/netsuite2/stg_netsuite2__transactions.sql | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 1e01d75..17d4daf 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -88,12 +88,16 @@ vars: transaction_lines_pass_through_columns: - name: "amortizstartdate" alias: "amortization_start_date" + transform_sql: "cast(amortization_start_date as date)" - name: "amortizationenddate" alias: "amortization_end_date" + transform_sql: "cast(amortization_end_date as date)" - name: "custcol_inv_start_date" alias: "lyra_inv_start_date" + transform_sql: "cast(lyra_inv_start_date as date)" - name: "custcol_inv_end_date" alias: "lyra_inv_end_date" + transform_sql: "cast(lyra_inv_end_date as date)" customers_pass_through_columns: [] locations_pass_through_columns: [] subsidiaries_pass_through_columns: diff --git a/models/netsuite2/stg_netsuite2__accounting_periods.sql b/models/netsuite2/stg_netsuite2__accounting_periods.sql index c63f392..8386b17 100644 --- a/models/netsuite2/stg_netsuite2__accounting_periods.sql +++ b/models/netsuite2/stg_netsuite2__accounting_periods.sql @@ -26,8 +26,8 @@ final as ( id as accounting_period_id, parent as parent_id, periodname as name, - startdate as starting_at, - enddate as ending_at, + cast(startdate as date) as starting_at, + cast(enddate as date) as ending_at, closedondate as closed_at, isquarter = 'T' as is_quarter, isyear = 'T' as is_year, diff --git a/models/netsuite2/stg_netsuite2__transactions.sql b/models/netsuite2/stg_netsuite2__transactions.sql index bb79a07..5eacdf0 100644 --- a/models/netsuite2/stg_netsuite2__transactions.sql +++ b/models/netsuite2/stg_netsuite2__transactions.sql @@ -27,10 +27,10 @@ final as ( transactionnumber as transaction_number, type as transaction_type, memo, - trandate as transaction_date, + cast(trandate as date) as transaction_date, status, createddate as created_at, - duedate as due_date_at, + cast(duedate as date) as due_date_at, closedate as closed_at, currency as currency_id, entity as entity_id, @@ -39,7 +39,7 @@ final as ( intercoadj = 'T' as is_intercompany_adjustment, isreversal = 'T' as is_reversal, reversal as reversal_transaction_id, - reversaldate as reversal_date, + cast(reversaldate as date) as reversal_date, reversaldefer = 'T' as is_reversal_defer --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model From ce8c3838343f0c7f3681b7cb954c3c7133edf37b Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:33:09 -0700 Subject: [PATCH 15/16] Revert pass through fields --- dbt_project.yml | 50 +++++-------------------------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index a63de03..a6ad972 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -57,54 +57,14 @@ vars: netsuite2_account_types: "{{ source('netsuite2', 'account_type') }}" netsuite2_entity_address: "{{ source('netsuite2', 'entity_address') }}" netsuite2_location_main_address: "{{ source('netsuite2', 'location_main_address') }}" - accounts_pass_through_columns: - - name: "custrecord_cash_flow_category" - alias: "lyra_cash_flow_category" - - name: "custrecord_ebitda_category" - alias: "lyra_ebitda_category" - - name: "custrecord_transfer_pricing_method" - alias: "lyra_transfer_pricing_method" + accounts_pass_through_columns: [] classes_pass_through_columns: [] - departments_pass_through_columns: - - name: "custrecord_sec_opex_category" - alias: "lyra_sec_opex_category" - transactions_pass_through_columns: - - name: "custbody_subledger" - alias: "lyra_subledger" - - name: "custbody_subledger_id" - alias: "lyra_subledger_id" - - name: "typebaseddocumentnumber" - alias: "type_based_document_number" - - name: "tranid" - alias: "tran_id" - - name: "createdby" - alias: "created_by_id" - - name: "createddate" - alias: "created_date" - - name: "lastmodifiedby" - alias: "last_modified_by_id" - - name: "lastmodifieddate" - alias: "last_modified_date" - transaction_lines_pass_through_columns: - - name: "amortizstartdate" - alias: "amortization_start_date" - transform_sql: "cast(amortization_start_date as date)" - - name: "amortizationenddate" - alias: "amortization_end_date" - transform_sql: "cast(amortization_end_date as date)" - - name: "custcol_inv_start_date" - alias: "lyra_inv_start_date" - transform_sql: "cast(lyra_inv_start_date as date)" - - name: "custcol_inv_end_date" - alias: "lyra_inv_end_date" - transform_sql: "cast(lyra_inv_end_date as date)" + departments_pass_through_columns: [] + transactions_pass_through_columns: [] + transaction_lines_pass_through_columns: [] customers_pass_through_columns: [] locations_pass_through_columns: [] - subsidiaries_pass_through_columns: - - name: "custrecord_transfer_pricing_markup" - alias: "lyra_transfer_pricing_markup" - - name: "custrecord_transfer_pricing_coa_method" - alias: "lyra_transfer_pricing_coa_method" + subsidiaries_pass_through_columns: [] consolidated_exchange_rates_pass_through_columns: [] vendors_pass_through_columns: [] items_pass_through_columns: [] From 423e1eb18f1c28c61bb548d529204f7324986a00 Mon Sep 17 00:00:00 2001 From: Jared Monger <95247581+jmongerlyra@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:51:51 -0700 Subject: [PATCH 16/16] Add back auto-release.yml --- .github/workflows/auto-release.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/auto-release.yml diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 0000000..8ed5853 --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,13 @@ +name: 'auto release' +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + call-workflow-passing-data: + if: github.event.pull_request.merged + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main + secrets: inherit \ No newline at end of file