Add employees model, fix timestamps, and add common fields

dbt_project.yml

@@ -41,6 +41,7 @@ vars:
     netsuite2_currencies: "{{ source('netsuite2','currency') }}"
     netsuite2_customers: "{{ source('netsuite2','customer') }}"
     netsuite2_departments: "{{ source('netsuite2','department') }}"
+    netsuite2_employees: "{{ source('netsuite2','employee') }}"
     netsuite2_entities: "{{ source('netsuite2','entity') }}"
     netsuite2_items: "{{ source('netsuite2','item') }}"
     netsuite2_jobs: "{{ source('netsuite2','job') }}"

macros/get_accounts_columns.sql

@@ -31,6 +31,7 @@
     {"name": "accttype", "datatype": dbt.type_string()},
     {"name": "sspecacct", "datatype": dbt.type_string()},
     {"name": "fullname", "datatype": dbt.type_string()},
+    {"name": "accountsearchdisplaynamecopy", "datatype": dbt.type_string()},
     {"name": "description", "datatype": dbt.type_string()},
     {"name": "deferralacct", "datatype": dbt.type_int()},
     {"name": "cashflowrate", "datatype": dbt.type_string()},

macros/get_customers_columns.sql

@@ -27,6 +27,7 @@
     {"name": "externalid", "datatype": dbt.type_string()},
     {"name": "parent", "datatype": dbt.type_int()},
     {"name": "isperson", "datatype": dbt.type_string()},
+    {"name": "altname", "datatype": dbt.type_string()},
     {"name": "companyname", "datatype": dbt.type_string()},
     {"name": "firstname", "datatype": dbt.type_string()},
     {"name": "lastname", "datatype": dbt.type_string()},

macros/get_employee_columns.sql

@@ -0,0 +1,35 @@
+{% macro get_employee_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "accountnumber", "datatype": dbt.type_string()},
+    {"name": "aliennumber", "datatype": dbt.type_string()},
+    {"name": "approvallimit", "datatype": dbt.type_float()},
+    {"name": "approver", "datatype": dbt.type_int()},
+    {"name": "class", "datatype": dbt.type_int()},
+    {"name": "comments", "datatype": dbt.type_string()},
+    {"name": "currency", "datatype": dbt.type_int()},
+    {"name": "department", "datatype": dbt.type_int()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "employeestatus", "datatype": dbt.type_int()},
+    {"name": "employeetype", "datatype": dbt.type_int()},
+    {"name": "entityid", "datatype": dbt.type_string()},
+    {"name": "expenselimit", "datatype": dbt.type_string()},
+    {"name": "firstname", "datatype": dbt.type_string()},
+    {"name": "giveaccess", "datatype": dbt.type_string()},
+    {"name": "hiredate", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "isinactive", "datatype": dbt.type_string()},
+    {"name": "lastname", "datatype": dbt.type_string()},
+    {"name": "location", "datatype": dbt.type_int()},
+    {"name": "middlename", "datatype": dbt.type_string()},
+    {"name": "purchaseorderapprovallimit", "datatype": dbt.type_float()},
+    {"name": "purchaseorderlimit", "datatype": dbt.type_float()},
+    {"name": "subsidiary", "datatype": dbt.type_int()},
+    {"name": "supervisor", "datatype": dbt.type_int()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_subsidiaries_columns.sql

@@ -77,6 +77,7 @@
     {"name": "state", "datatype": dbt.type_string()},
     {"name": "fiscalcalendar", "datatype": dbt.type_int()},
     {"name": "parent", "datatype": dbt.type_int()},
+    {"name": "iselimination", "datatype": dbt.type_string()},
     {"name": "currency", "datatype": dbt.type_int()}
 ] %}
 

macros/get_transaction_lines_columns.sql

@@ -42,7 +42,9 @@
     {"name": "cleared", "datatype": dbt.type_string()},
     {"name": "commitmentfirm", "datatype": dbt.type_string()},
     {"name": "mainline", "datatype": dbt.type_string()},
-    {"name": "taxline", "datatype": dbt.type_string()}
+    {"name": "taxline", "datatype": dbt.type_string()},
+    {"name": "eliminate", "datatype": dbt.type_string()},
+    {"name": "netamount", "datatype": dbt.type_float()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('transaction_lines_pass_through_columns')) }}

macros/get_transactions_columns.sql

@@ -29,15 +29,20 @@
     {"name": "memo", "datatype": dbt.type_string()},
     {"name": "trandate", "datatype": dbt.type_timestamp()},
     {"name": "status", "datatype": dbt.type_string()},
+    {"name": "createdby", "datatype": dbt.type_int()},
     {"name": "createddate", "datatype": dbt.type_timestamp()},
     {"name": "duedate", "datatype": dbt.type_timestamp()},
     {"name": "closedate", "datatype": dbt.type_timestamp()},
     {"name": "currency", "datatype": dbt.type_int()},
     {"name": "entity", "datatype": dbt.type_int()},
+    {"name": "lastmodifiedby", "datatype": dbt.type_int()},
     {"name": "postingperiod", "datatype": dbt.type_int()},
     {"name": "posting", "datatype": dbt.type_string()},
     {"name": "intercoadj", "datatype": dbt.type_string()},
     {"name": "isreversal", "datatype": dbt.type_string()},
+    {"name": "reversal", "datatype": dbt.type_int()},
+    {"name": "reversaldate", "datatype": dbt.type_timestamp()},
+    {"name": "reversaldefer", "datatype": dbt.type_string()}, 
     {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 

models/docs.md

@@ -36,6 +36,8 @@
 
 {% docs entities_table %} Table detailing all entities in Netsuite. {% enddocs %}
 
+{% docs employee_table %} Table detailing all employees in Netsuite. {% enddocs %}
+
 {% docs jobs_table %} Table detailing all jobs. {% enddocs %}
 
 {% docs transaction_accounting_lines_table %} A table detailing all transaction lines for all transactions. {% enddocs %}
@@ -61,6 +63,8 @@
 
 {% docs vendor_id %} The unique identifier of the vendor. {% enddocs %}
 
+{% docs alt_name %} Name of the person or company. {% enddocs %}
+
 {% docs company_name %} Name of the company. {% enddocs %}
 
 {% docs create_date_at %} Timestamp of the record creation. {% enddocs %}
@@ -75,6 +79,10 @@
 
 {% docs entity_id %} The entity id of the entity used for the record. {% enddocs %}
 
+{% docs employee_id %} The employee id of the employee used for the record. {% enddocs %}
+
+{% docs supervisor_id %} The employee id of the employee's supervisor used for the record. {% enddocs %}
+
 {% docs transaction_id %} The transaction id of referenced for the record. {% enddocs %}
 
 {% docs department_id %} The unique identifier of the department used for the record. {% enddocs %}

models/netsuite2/src_netsuite2.yml

@@ -286,6 +286,41 @@ sources:
           - name: _fivetran_deleted
             description: "{{ doc('_fivetran_deleted') }}"
 
+      - name: employee
+        identified: "{{ var('netsuite2_employee_identifier', 'employee') }}"
+        description: "{{ doc('employee_table') }}"
+        columns:
+          - name: _fivetran_synced
+            description: "{{ doc('_fivetran_synced') }}"
+          - name: id
+            description: "{{ doc('employee_id') }}"
+          - name: entityid
+            description: "{{ doc('entity_id') }}"
+          - name: firstname
+            description: First name of the employee.
+          - name: lastname
+            description: Last name of the employee.
+          - name: department
+            description: "{{ doc('department_id') }}"
+          - name: subsidiary
+            description: "{{ doc('subsidiary_id') }}"
+          - name: email
+            description: Email address of the employee.
+          - name: supervisor
+            description: "{{ doc('supervisor_id') }}"
+          - name: approvallimit
+            description: The largest expense report this supervisor can approve.
+          - name: expenselimit
+            description: Amount this employee can expense without approval from a supervisor or approver.
+          - name: purchaseorderapprovallimit
+            description: Amount an employee is allowed to approve on a purchase request when specified as an approver.
+          - name: purchaseorderlimit
+            description: Amount this employee can purchase without approval from a supervisor or approver.
+          - name: currency
+            description: "{{ doc('currency_id') }}"
+          - name: isinactive
+            description: Boolean indicating this employee is active.
+
       - name: entity
         identifier: "{{ var('netsuite2_entity_identifier', 'entity') }}"
         description: "{{ doc('entities_table') }}"
@@ -472,6 +507,8 @@ sources:
             description: Reference to the parent subsidiary.
           - name: currency
             description:  "{{ doc('currency_id') }}"
+          - name: iselimination
+            description: Boolean indicating if the subsidiary is an elimination subsidiary. Represented as "T" or "F" for true and false respectively.
 
       - name: transaction_accounting_line
         identifier: "{{ var('netsuite2_transaction_accounting_line_identifier', 'transactionaccountingline') }}"
@@ -540,6 +577,10 @@ sources:
             description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively.
           - name: taxline
             description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively.
+          - name: eliminate
+            description: Boolean indicating if the transaction line is eliminated. Represented as "T" or "F" for true and false respectively.
+          - name: netamount
+            description: Net amount of transaction line.
 
       - name: transaction
         identifier: "{{ var('netsuite2_transaction_identifier', 'transaction') }}"

models/netsuite2/stg_netsuite2__accounting_periods.sql

@@ -26,8 +26,8 @@ final as (
         id as accounting_period_id,
         parent as parent_id, 
         periodname as name,
-        startdate as starting_at,
-        enddate as ending_at,
+        cast(startdate as date) as starting_at,
+        cast(enddate as date) as ending_at,
         closedondate as closed_at,
         isquarter = 'T' as is_quarter,
         isyear = 'T' as is_year,

models/netsuite2/stg_netsuite2__accounts.sql

@@ -29,6 +29,7 @@ final as (
         accttype as account_type_id,
         sspecacct as special_account_type_id,
         fullname as name,
+        accountsearchdisplaynamecopy as display_name,
         description as account_description,
         deferralacct as deferral_account_id,
         cashflowrate as cash_flow_rate_type,

models/netsuite2/stg_netsuite2__customers.sql

@@ -26,6 +26,7 @@ final as (
         externalid as customer_external_id,
         parent as parent_id,
         isperson = 'T' as is_person,
+        altname as alt_name,
         companyname as company_name,
         firstname as first_name,
         lastname as last_name,

models/netsuite2/stg_netsuite2__employees.sql

@@ -0,0 +1,43 @@
+{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }}
+
+with base as (
+
+    select * 
+    from {{ ref('stg_netsuite2__employees_tmp') }}
+),
+
+fields as (
+
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_netsuite2__employees_tmp')),
+                staging_columns=get_employee_columns()
+            )
+        }}
+    from base
+),
+
+final as (
+
+    select
+        _fivetran_synced,
+        id as employee_id,
+        entityid as entity_id,
+        firstname as first_name,
+        lastname as last_name,
+        department as department_id,
+        subsidiary as subsidiary_id,
+        email,
+        supervisor as supervisor_id,
+        approvallimit as approval_limit,
+        expenselimit as expense_limit,
+        purchaseorderapprovallimit as purchase_order_approval_limit,
+        purchaseorderlimit as purchase_order_limit,
+        currency as currency_id,
+        isinactive = 'T' as is_inactive
+    from fields
+)
+
+select * 
+from final

models/netsuite2/stg_netsuite2__subsidiaries.sql

@@ -31,6 +31,7 @@ final as (
         state,
         fiscalcalendar as fiscal_calendar_id,
         parent as parent_id,
+        iselimination = 'T' as is_elimination,
         currency as currency_id
 
         --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model

models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql

@@ -28,6 +28,7 @@ final as (
         accountingbook as accounting_book_id,
         account as account_id,
         posting = 'T' as is_posting,
+        exchangerate as exchange_rate,
         amount,
         credit as credit_amount,
         debit as debit_amount,

models/netsuite2/stg_netsuite2__transaction_lines.sql

@@ -38,7 +38,9 @@ final as (
         cleared = 'T' as is_cleared,
         commitmentfirm = 'T' as is_commitment_firm,
         mainline = 'T' as is_main_line,
-        taxline = 'T' as is_tax_line
+        taxline = 'T' as is_tax_line,
+        eliminate = 'T' as is_eliminate,
+        netamount
 
         --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }}

models/netsuite2/stg_netsuite2__transactions.sql

@@ -27,17 +27,20 @@ final as (
         transactionnumber as transaction_number,
         type as transaction_type,
         memo,
-        trandate as transaction_date,
+        cast(trandate as date) as transaction_date,
         status,
         createddate as created_at,
-        duedate as due_date_at,
+        cast(duedate as date) as due_date_at,
         closedate as closed_at,
         currency as currency_id,
         entity as entity_id,
         postingperiod as accounting_period_id,
         posting = 'T' as is_posting,
         intercoadj = 'T' as is_intercompany_adjustment,
-        isreversal = 'T' as is_reversal
+        isreversal = 'T' as is_reversal,
+        reversal as reversal_transaction_id,
+        cast(reversaldate as date) as reversal_date,
+        reversaldefer = 'T' as is_reversal_defer
 
         --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('transactions_pass_through_columns') }}

models/netsuite2/stg_netsuite2__vendors.sql

@@ -24,6 +24,7 @@ final as (
     select
         _fivetran_synced,
         id as vendor_id,
+        altname as alt_name,
         companyname as company_name,
         datecreated as create_date_at,
         category as vendor_category_id

models/netsuite2/tmp/stg_netsuite2__employees_tmp.sql

@@ -0,0 +1,4 @@
+{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }}
+
+select * 
+from {{ var('netsuite2_employees') }}