diff --git a/dbt_project.yml b/dbt_project.yml index cc7442e..a6ad972 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -41,6 +41,7 @@ vars: netsuite2_currencies: "{{ source('netsuite2','currency') }}" netsuite2_customers: "{{ source('netsuite2','customer') }}" netsuite2_departments: "{{ source('netsuite2','department') }}" + netsuite2_employees: "{{ source('netsuite2','employee') }}" netsuite2_entities: "{{ source('netsuite2','entity') }}" netsuite2_items: "{{ source('netsuite2','item') }}" netsuite2_jobs: "{{ source('netsuite2','job') }}" diff --git a/macros/get_accounts_columns.sql b/macros/get_accounts_columns.sql index 1771adb..66fc972 100644 --- a/macros/get_accounts_columns.sql +++ b/macros/get_accounts_columns.sql @@ -31,6 +31,7 @@ {"name": "accttype", "datatype": dbt.type_string()}, {"name": "sspecacct", "datatype": dbt.type_string()}, {"name": "fullname", "datatype": dbt.type_string()}, + {"name": "accountsearchdisplaynamecopy", "datatype": dbt.type_string()}, {"name": "description", "datatype": dbt.type_string()}, {"name": "deferralacct", "datatype": dbt.type_int()}, {"name": "cashflowrate", "datatype": dbt.type_string()}, diff --git a/macros/get_customers_columns.sql b/macros/get_customers_columns.sql index a8f2542..7a6a9f5 100644 --- a/macros/get_customers_columns.sql +++ b/macros/get_customers_columns.sql @@ -27,6 +27,7 @@ {"name": "externalid", "datatype": dbt.type_string()}, {"name": "parent", "datatype": dbt.type_int()}, {"name": "isperson", "datatype": dbt.type_string()}, + {"name": "altname", "datatype": dbt.type_string()}, {"name": "companyname", "datatype": dbt.type_string()}, {"name": "firstname", "datatype": dbt.type_string()}, {"name": "lastname", "datatype": dbt.type_string()}, diff --git a/macros/get_employee_columns.sql b/macros/get_employee_columns.sql new file mode 100644 index 0000000..de41b6f --- /dev/null +++ b/macros/get_employee_columns.sql @@ -0,0 +1,35 @@ +{% macro get_employee_columns() %} + +{% set columns = [ + {"name": "_fivetran_deleted", "datatype": "boolean"}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "accountnumber", "datatype": dbt.type_string()}, + {"name": "aliennumber", "datatype": dbt.type_string()}, + {"name": "approvallimit", "datatype": dbt.type_float()}, + {"name": "approver", "datatype": dbt.type_int()}, + {"name": "class", "datatype": dbt.type_int()}, + {"name": "comments", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_int()}, + {"name": "department", "datatype": dbt.type_int()}, + {"name": "email", "datatype": dbt.type_string()}, + {"name": "employeestatus", "datatype": dbt.type_int()}, + {"name": "employeetype", "datatype": dbt.type_int()}, + {"name": "entityid", "datatype": dbt.type_string()}, + {"name": "expenselimit", "datatype": dbt.type_string()}, + {"name": "firstname", "datatype": dbt.type_string()}, + {"name": "giveaccess", "datatype": dbt.type_string()}, + {"name": "hiredate", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_int()}, + {"name": "isinactive", "datatype": dbt.type_string()}, + {"name": "lastname", "datatype": dbt.type_string()}, + {"name": "location", "datatype": dbt.type_int()}, + {"name": "middlename", "datatype": dbt.type_string()}, + {"name": "purchaseorderapprovallimit", "datatype": dbt.type_float()}, + {"name": "purchaseorderlimit", "datatype": dbt.type_float()}, + {"name": "subsidiary", "datatype": dbt.type_int()}, + {"name": "supervisor", "datatype": dbt.type_int()} +] %} + +{{ return(columns) }} + +{% endmacro %} diff --git a/macros/get_subsidiaries_columns.sql b/macros/get_subsidiaries_columns.sql index 3466d73..e77ad1e 100644 --- a/macros/get_subsidiaries_columns.sql +++ b/macros/get_subsidiaries_columns.sql @@ -77,6 +77,7 @@ {"name": "state", "datatype": dbt.type_string()}, {"name": "fiscalcalendar", "datatype": dbt.type_int()}, {"name": "parent", "datatype": dbt.type_int()}, + {"name": "iselimination", "datatype": dbt.type_string()}, {"name": "currency", "datatype": dbt.type_int()} ] %} diff --git a/macros/get_transaction_lines_columns.sql b/macros/get_transaction_lines_columns.sql index a2cab97..b1629d7 100644 --- a/macros/get_transaction_lines_columns.sql +++ b/macros/get_transaction_lines_columns.sql @@ -42,7 +42,9 @@ {"name": "cleared", "datatype": dbt.type_string()}, {"name": "commitmentfirm", "datatype": dbt.type_string()}, {"name": "mainline", "datatype": dbt.type_string()}, - {"name": "taxline", "datatype": dbt.type_string()} + {"name": "taxline", "datatype": dbt.type_string()}, + {"name": "eliminate", "datatype": dbt.type_string()}, + {"name": "netamount", "datatype": dbt.type_float()} ] %} {{ fivetran_utils.add_pass_through_columns(columns, var('transaction_lines_pass_through_columns')) }} diff --git a/macros/get_transactions_columns.sql b/macros/get_transactions_columns.sql index c3d180a..1ace50d 100644 --- a/macros/get_transactions_columns.sql +++ b/macros/get_transactions_columns.sql @@ -29,15 +29,20 @@ {"name": "memo", "datatype": dbt.type_string()}, {"name": "trandate", "datatype": dbt.type_timestamp()}, {"name": "status", "datatype": dbt.type_string()}, + {"name": "createdby", "datatype": dbt.type_int()}, {"name": "createddate", "datatype": dbt.type_timestamp()}, {"name": "duedate", "datatype": dbt.type_timestamp()}, {"name": "closedate", "datatype": dbt.type_timestamp()}, {"name": "currency", "datatype": dbt.type_int()}, {"name": "entity", "datatype": dbt.type_int()}, + {"name": "lastmodifiedby", "datatype": dbt.type_int()}, {"name": "postingperiod", "datatype": dbt.type_int()}, {"name": "posting", "datatype": dbt.type_string()}, {"name": "intercoadj", "datatype": dbt.type_string()}, {"name": "isreversal", "datatype": dbt.type_string()}, + {"name": "reversal", "datatype": dbt.type_int()}, + {"name": "reversaldate", "datatype": dbt.type_timestamp()}, + {"name": "reversaldefer", "datatype": dbt.type_string()}, {"name": "_fivetran_deleted", "datatype": "boolean"} ] %} diff --git a/models/docs.md b/models/docs.md index b18b185..33d641d 100644 --- a/models/docs.md +++ b/models/docs.md @@ -36,6 +36,8 @@ {% docs entities_table %} Table detailing all entities in Netsuite. {% enddocs %} +{% docs employee_table %} Table detailing all employees in Netsuite. {% enddocs %} + {% docs jobs_table %} Table detailing all jobs. {% enddocs %} {% docs transaction_accounting_lines_table %} A table detailing all transaction lines for all transactions. {% enddocs %} @@ -61,6 +63,8 @@ {% docs vendor_id %} The unique identifier of the vendor. {% enddocs %} +{% docs alt_name %} Name of the person or company. {% enddocs %} + {% docs company_name %} Name of the company. {% enddocs %} {% docs create_date_at %} Timestamp of the record creation. {% enddocs %} @@ -75,6 +79,10 @@ {% docs entity_id %} The entity id of the entity used for the record. {% enddocs %} +{% docs employee_id %} The employee id of the employee used for the record. {% enddocs %} + +{% docs supervisor_id %} The employee id of the employee's supervisor used for the record. {% enddocs %} + {% docs transaction_id %} The transaction id of referenced for the record. {% enddocs %} {% docs department_id %} The unique identifier of the department used for the record. {% enddocs %} diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml index bfbc7d3..6aeb958 100644 --- a/models/netsuite2/src_netsuite2.yml +++ b/models/netsuite2/src_netsuite2.yml @@ -286,6 +286,41 @@ sources: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: employee + identified: "{{ var('netsuite2_employee_identifier', 'employee') }}" + description: "{{ doc('employee_table') }}" + columns: + - name: _fivetran_synced + description: "{{ doc('_fivetran_synced') }}" + - name: id + description: "{{ doc('employee_id') }}" + - name: entityid + description: "{{ doc('entity_id') }}" + - name: firstname + description: First name of the employee. + - name: lastname + description: Last name of the employee. + - name: department + description: "{{ doc('department_id') }}" + - name: subsidiary + description: "{{ doc('subsidiary_id') }}" + - name: email + description: Email address of the employee. + - name: supervisor + description: "{{ doc('supervisor_id') }}" + - name: approvallimit + description: The largest expense report this supervisor can approve. + - name: expenselimit + description: Amount this employee can expense without approval from a supervisor or approver. + - name: purchaseorderapprovallimit + description: Amount an employee is allowed to approve on a purchase request when specified as an approver. + - name: purchaseorderlimit + description: Amount this employee can purchase without approval from a supervisor or approver. + - name: currency + description: "{{ doc('currency_id') }}" + - name: isinactive + description: Boolean indicating this employee is active. + - name: entity identifier: "{{ var('netsuite2_entity_identifier', 'entity') }}" description: "{{ doc('entities_table') }}" @@ -472,6 +507,8 @@ sources: description: Reference to the parent subsidiary. - name: currency description: "{{ doc('currency_id') }}" + - name: iselimination + description: Boolean indicating if the subsidiary is an elimination subsidiary. Represented as "T" or "F" for true and false respectively. - name: transaction_accounting_line identifier: "{{ var('netsuite2_transaction_accounting_line_identifier', 'transactionaccountingline') }}" @@ -540,6 +577,10 @@ sources: description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively. - name: taxline description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. + - name: eliminate + description: Boolean indicating if the transaction line is eliminated. Represented as "T" or "F" for true and false respectively. + - name: netamount + description: Net amount of transaction line. - name: transaction identifier: "{{ var('netsuite2_transaction_identifier', 'transaction') }}" diff --git a/models/netsuite2/stg_netsuite2__accounting_periods.sql b/models/netsuite2/stg_netsuite2__accounting_periods.sql index c63f392..8386b17 100644 --- a/models/netsuite2/stg_netsuite2__accounting_periods.sql +++ b/models/netsuite2/stg_netsuite2__accounting_periods.sql @@ -26,8 +26,8 @@ final as ( id as accounting_period_id, parent as parent_id, periodname as name, - startdate as starting_at, - enddate as ending_at, + cast(startdate as date) as starting_at, + cast(enddate as date) as ending_at, closedondate as closed_at, isquarter = 'T' as is_quarter, isyear = 'T' as is_year, diff --git a/models/netsuite2/stg_netsuite2__accounts.sql b/models/netsuite2/stg_netsuite2__accounts.sql index f55b702..f4bd496 100644 --- a/models/netsuite2/stg_netsuite2__accounts.sql +++ b/models/netsuite2/stg_netsuite2__accounts.sql @@ -29,6 +29,7 @@ final as ( accttype as account_type_id, sspecacct as special_account_type_id, fullname as name, + accountsearchdisplaynamecopy as display_name, description as account_description, deferralacct as deferral_account_id, cashflowrate as cash_flow_rate_type, diff --git a/models/netsuite2/stg_netsuite2__customers.sql b/models/netsuite2/stg_netsuite2__customers.sql index 8a9fca1..cc3303e 100644 --- a/models/netsuite2/stg_netsuite2__customers.sql +++ b/models/netsuite2/stg_netsuite2__customers.sql @@ -26,6 +26,7 @@ final as ( externalid as customer_external_id, parent as parent_id, isperson = 'T' as is_person, + altname as alt_name, companyname as company_name, firstname as first_name, lastname as last_name, diff --git a/models/netsuite2/stg_netsuite2__employees.sql b/models/netsuite2/stg_netsuite2__employees.sql new file mode 100644 index 0000000..b5a0d61 --- /dev/null +++ b/models/netsuite2/stg_netsuite2__employees.sql @@ -0,0 +1,43 @@ +{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} + +with base as ( + + select * + from {{ ref('stg_netsuite2__employees_tmp') }} +), + +fields as ( + + select + {{ + fivetran_utils.fill_staging_columns( + source_columns=adapter.get_columns_in_relation(ref('stg_netsuite2__employees_tmp')), + staging_columns=get_employee_columns() + ) + }} + from base +), + +final as ( + + select + _fivetran_synced, + id as employee_id, + entityid as entity_id, + firstname as first_name, + lastname as last_name, + department as department_id, + subsidiary as subsidiary_id, + email, + supervisor as supervisor_id, + approvallimit as approval_limit, + expenselimit as expense_limit, + purchaseorderapprovallimit as purchase_order_approval_limit, + purchaseorderlimit as purchase_order_limit, + currency as currency_id, + isinactive = 'T' as is_inactive + from fields +) + +select * +from final diff --git a/models/netsuite2/stg_netsuite2__subsidiaries.sql b/models/netsuite2/stg_netsuite2__subsidiaries.sql index fb5b0e2..50bb280 100644 --- a/models/netsuite2/stg_netsuite2__subsidiaries.sql +++ b/models/netsuite2/stg_netsuite2__subsidiaries.sql @@ -31,6 +31,7 @@ final as ( state, fiscalcalendar as fiscal_calendar_id, parent as parent_id, + iselimination = 'T' as is_elimination, currency as currency_id --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model diff --git a/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql b/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql index 2a6dfb0..2eb5fc8 100644 --- a/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql +++ b/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql @@ -28,6 +28,7 @@ final as ( accountingbook as accounting_book_id, account as account_id, posting = 'T' as is_posting, + exchangerate as exchange_rate, amount, credit as credit_amount, debit as debit_amount, diff --git a/models/netsuite2/stg_netsuite2__transaction_lines.sql b/models/netsuite2/stg_netsuite2__transaction_lines.sql index a6f79b4..ba24b12 100644 --- a/models/netsuite2/stg_netsuite2__transaction_lines.sql +++ b/models/netsuite2/stg_netsuite2__transaction_lines.sql @@ -38,7 +38,9 @@ final as ( cleared = 'T' as is_cleared, commitmentfirm = 'T' as is_commitment_firm, mainline = 'T' as is_main_line, - taxline = 'T' as is_tax_line + taxline = 'T' as is_tax_line, + eliminate = 'T' as is_eliminate, + netamount --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }} diff --git a/models/netsuite2/stg_netsuite2__transactions.sql b/models/netsuite2/stg_netsuite2__transactions.sql index a9384ff..5eacdf0 100644 --- a/models/netsuite2/stg_netsuite2__transactions.sql +++ b/models/netsuite2/stg_netsuite2__transactions.sql @@ -27,17 +27,20 @@ final as ( transactionnumber as transaction_number, type as transaction_type, memo, - trandate as transaction_date, + cast(trandate as date) as transaction_date, status, createddate as created_at, - duedate as due_date_at, + cast(duedate as date) as due_date_at, closedate as closed_at, currency as currency_id, entity as entity_id, postingperiod as accounting_period_id, posting = 'T' as is_posting, intercoadj = 'T' as is_intercompany_adjustment, - isreversal = 'T' as is_reversal + isreversal = 'T' as is_reversal, + reversal as reversal_transaction_id, + cast(reversaldate as date) as reversal_date, + reversaldefer = 'T' as is_reversal_defer --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('transactions_pass_through_columns') }} diff --git a/models/netsuite2/stg_netsuite2__vendors.sql b/models/netsuite2/stg_netsuite2__vendors.sql index 22a26bb..a5c2d78 100644 --- a/models/netsuite2/stg_netsuite2__vendors.sql +++ b/models/netsuite2/stg_netsuite2__vendors.sql @@ -24,6 +24,7 @@ final as ( select _fivetran_synced, id as vendor_id, + altname as alt_name, companyname as company_name, datecreated as create_date_at, category as vendor_category_id diff --git a/models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql new file mode 100644 index 0000000..bb16dae --- /dev/null +++ b/models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql @@ -0,0 +1,4 @@ +{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} + +select * +from {{ var('netsuite2_employees') }} diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml index 6acc382..02b3a13 100644 --- a/models/stg_netsuite2.yml +++ b/models/stg_netsuite2.yml @@ -246,6 +246,8 @@ models: description: Reference to the parent customer. - name: is_person description: Boolean indicating if the customer is an individual person. + - name: alt_name + description: "{{ doc('alt_name') }}" - name: company_name description: "{{ doc('company_name') }}" - name: first_name @@ -290,6 +292,43 @@ models: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: stg_netsuite2__employees + description: "{{ doc('employee_table') }}" + columns: + - name: _fivetran_synced + description: "{{ doc('_fivetran_synced') }}" + - name: employee_id + description: "{{ doc('employee_id') }}" + tests: + - unique + - not_null + - name: entity_id + description: "{{ doc('entity_id') }}" + - name: first_name + description: First name of the employee. + - name: last_name + description: Last name of the employee. + - name: department_id + description: "{{ doc('department_id') }}" + - name: subsidiary_id + description: "{{ doc('subsidiary_id') }}" + - name: email + description: Email address of the employee. + - name: supervisor_id + description: "{{ doc('supervisor_id') }}" + - name: approval_limit + description: The largest expense report this supervisor can approve. + - name: expense_limit + description: Amount this employee can expense without approval from a supervisor or approver. + - name: purchase_order_approval_limit + description: Amount an employee is allowed to approve on a purchase request when specified as an approver. + - name: purchase_order_limit + description: Amount this employee can purchase without approval from a supervisor or approver. + - name: currency_id + description: "{{ doc('currency_id') }}" + - name: is_inactive + description: Boolean indicating this employee is active. + - name: stg_netsuite2__entities description: "{{ doc('entities_table') }}" columns: @@ -482,6 +521,8 @@ models: description: "{{ doc('fiscal_calendar_id') }}" - name: parent_id description: Reference to the parent subsidiary. + - name: is_elimination + description: Boolean indicating if the subsidiary is an elimination subsidiary. - name: currency_id description: "{{ doc('currency_id') }}" @@ -573,6 +614,10 @@ models: description: Boolean indicating if the transaction line is a main line entry. - name: is_tax_line description: Boolean indicating if the transaction line is a tax line. + - name: is_eliminate + description: Boolean indicating if the transaction line is eliminated. + - name: netamount + description: Net amount of transaction line. - name: stg_netsuite2__transactions description: "{{ doc('transaction_table') }}" @@ -638,6 +683,8 @@ models: tests: - unique - not_null + - name: alt_name + description: "{{ doc('alt_name') }}" - name: company_name description: "{{ doc('company_name') }}" - name: create_date_at