Merge pull request #95 from fivetran/bugfix/join_on_source_relation

Bugfix/join on source relation

.gitignore

@@ -3,5 +3,5 @@ target/
 dbt_modules/
 logs/
 .DS_Store
-integration_test/.DS_Store
+integration_tests/.DS_Store
 dbt_packages/

CHANGELOG.md

@@ -1,3 +1,20 @@
+# dbt_quickbooks v0.11.0
+## 🚨 Breaking Changes 🚨
+[PR #95](https://github.com/fivetran/dbt_quickbooks/pull/95) includes the following updates:
+## 🪲 Bug Fixes
+- Included `source_relation` in all joins and window functions for models outputting `source_relation`. This is to prevent duplicate records in end models when using the unioning functionality. These updates were in the intermediate models, which flowed to downstream end models:
+  - `quickbooks__general_ledger`
+  - `quickbooks__expenses_sales_enhanced`
+- In end model `quickbooks__general_ledger`, added `source_relation` as part of the generated surrogate key `unique_id` to prevent duplicate `unique_id`s when using the unioning functionality.
+
+## 🎉 Features
+- Added description for column `source_relation` to the documentation.
+
+## 🚘 Under the Hood
+- Updated test from a combination of columns to uniqueness of `unique_id` in `quickbooks__general_ledger`. 
+- Updated partitioning in certain models to include `source_relation`. 
+- Updated analysis `quickbooks__balance_sheet` with updated join strategy. 
+
 # dbt_quickbooks v0.10.0
 ## 🎉 Feature Update 🎉
 - Databricks compatibility! ([#92](https://github.com/fivetran/dbt_quickbooks/pull/92))

README.md

@@ -71,7 +71,7 @@ Include the following QuickBooks package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/quickbooks
-    version: [">=0.10.0", "<0.11.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.11.0", "<0.12.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do NOT include the `quickbooks_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.

analysis/quickbooks__balance_sheet.sql

@@ -43,25 +43,28 @@ liability as (
         l.period_ending_balance
     from liability_date ld
 
-    left join (select account_id, period_first_day, period_ending_balance from general_ledger_by_period where account_class = 'Liability') l
+    left join (select account_id, source_relation, period_first_day, period_ending_balance from general_ledger_by_period where account_class = 'Liability') l
         on l.account_id = ld.account_id
-            and l.period_first_day = ld.period_first_day 
+            and l.period_first_day = ld.period_first_day
+            and l.source_relation = ld.source_relation
 ),
 
 asset as (
     select ad.account_id, a.period_ending_balance
     from asset_date ad
-        left join (select account_id, period_first_day, period_ending_balance from general_ledger_by_period where account_class = 'Asset') a
+        left join (select account_id, source_relation, period_first_day, period_ending_balance from general_ledger_by_period where account_class = 'Asset') a
             on a.account_id = ad.account_id
                 and a.period_first_day = ad.period_first_day 
+                and a.source_relation = ad.source_relation
 ),
 
 equity as (
     select ed.account_id, e.period_ending_balance
     from equity_date ed
-        left join (select account_id, period_first_day, period_ending_balance from general_ledger_by_period where account_class = 'Equity') e
+        left join (select account_id, source_relation, period_first_day, period_ending_balance from general_ledger_by_period where account_class = 'Equity') e
             on e.account_id = ed.account_id
                 and e.period_first_day = ed.period_first_day 
+                and e.source_relation = ed.source_relation
 )
 
 select 

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 name: 'quickbooks'
 
-version: '0.10.0'
+version: '0.11.0'
 
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 name: 'quickbooks_integration_tests'
 
-version: '0.10.0'
+version: '0.11.0'
 
 profile: 'integration_tests'
 config-version: 2

integration_tests/requirements.txt

@@ -1,6 +1,6 @@
 dbt-snowflake>=1.3.0,<2.0.0
 dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<2.0.0
+dbt-redshift>=1.3.0,<1.5.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0

models/docs.md

@@ -67,6 +67,10 @@ The parent account name. If the account is the parent account then the account n
 The parent account number. If the account is the parent account then the account number is recorded.
 {% enddocs %}
 
+{% docs source_relation %}
+The source of the record if the unioning functionality is being used. If not this field will be null. 
+{% enddocs %}
+
 {% docs transaction_date %}
 Timestamp of the date that the transaction occurred.
 {% enddocs %}

models/double_entry_transactions/int_quickbooks__bill_double_entry.sql

@@ -27,6 +27,7 @@ items as (
 
     left join {{ ref('stg_quickbooks__item') }} parent
         on item.parent_item_id = parent.item_id
+        and item.source_relation = parent.source_relation
 ),
 
 bill_join as (

models/double_entry_transactions/int_quickbooks__bill_payment_double_entry.sql

@@ -26,7 +26,8 @@ accounts as (
 ap_accounts as (
 
     select
-        account_id
+        account_id,
+        source_relation
     from accounts
 
     where account_type = 'Accounts Payable'
@@ -38,7 +39,8 @@ bill_payment_join as (
     select
         bill_payments.bill_payment_id as transaction_id,
         bill_payments.source_relation,
-        row_number() over(partition by bill_payments.bill_payment_id order by bill_payments.transaction_date) - 1 as index,
+        row_number() over(partition by bill_payments.bill_payment_id, bill_payments.source_relation 
+            order by bill_payments.source_relation, bill_payments.transaction_date) - 1 as index,
         bill_payments.transaction_date,
         bill_payments.total_amount as amount,
         coalesce(bill_payments.credit_card_account_id,bill_payments.check_bank_account_id) as payment_account_id,
@@ -47,7 +49,8 @@ bill_payment_join as (
         bill_payments.department_id
     from bill_payments
 
-    cross join ap_accounts
+    inner join ap_accounts
+    on ap_accounts.source_relation = bill_payments.source_relation
 
 ),
 

models/double_entry_transactions/int_quickbooks__credit_card_pymt_double_entry.sql

@@ -17,7 +17,8 @@ credit_card_payment_prep as (
     select
         credit_card_payments.credit_card_payment_id as transaction_id,
         credit_card_payments.source_relation,
-        row_number() over (partition by credit_card_payments.credit_card_payment_id order by credit_card_payments.transaction_date) - 1 as index,
+        row_number() over (partition by credit_card_payments.credit_card_payment_id, credit_card_payments.source_relation 
+            order by credit_card_payments.source_relation, credit_card_payments.transaction_date) - 1 as index,
         credit_card_payments.transaction_date,
         credit_card_payments.amount,
         credit_card_payments.bank_account_id,

models/double_entry_transactions/int_quickbooks__credit_memo_double_entry.sql

@@ -32,7 +32,8 @@ accounts as (
 df_accounts as (
 
     select
-        account_id as account_id
+        account_id as account_id,
+        source_relation
     from accounts
 
     where account_type = 'Accounts Receivable'
@@ -70,7 +71,7 @@ final as (
 
     select
         transaction_id,
-        source_relation,
+        credit_memo_join.source_relation,
         index,
         transaction_date,
         customer_id,
@@ -87,7 +88,7 @@ final as (
 
     select
         transaction_id,
-        source_relation,
+        credit_memo_join.source_relation,
         index,
         transaction_date,
         customer_id,
@@ -100,7 +101,8 @@ final as (
         'credit_memo' as transaction_source
     from credit_memo_join
 
-    cross join df_accounts
+    inner join df_accounts
+    on df_accounts.source_relation = credit_memo_join.source_relation
 )
 
 select *

models/double_entry_transactions/int_quickbooks__deposit_double_entry.sql

@@ -27,7 +27,8 @@ accounts as (
 uf_accounts as (
 
     select
-        account_id
+        account_id,
+        source_relation
     from accounts
 
     where account_sub_type = 'UndepositedFunds'
@@ -55,7 +56,8 @@ deposit_join as (
         on deposits.deposit_id = deposit_lines.deposit_id
         and deposits.source_relation = deposit_lines.source_relation
 
-    cross join uf_accounts
+    inner join uf_accounts
+    on uf_accounts.source_relation = deposits.source_relation
 
 ),
 

models/double_entry_transactions/int_quickbooks__invoice_double_entry.sql

@@ -85,8 +85,10 @@ bundle_income_accounts as (
 
 ar_accounts as (
 
-    select *
-    from {{ ref('stg_quickbooks__account') }}
+    select 
+        account_id,
+        source_relation
+    from accounts
 
     where account_type = 'Accounts Receivable'
 ),
@@ -190,7 +192,8 @@ final as (
         end as transaction_source
     from invoice_filter
 
-    cross join ar_accounts
+    inner join ar_accounts
+    on ar_accounts.source_relation = invoice_filter.source_relation
 )
 
 select *

models/double_entry_transactions/int_quickbooks__payment_double_entry.sql

@@ -26,7 +26,8 @@ accounts as (
 ar_accounts as (
 
     select
-        account_id
+        account_id,
+        source_relation
     from accounts
 
     where account_type = 'Accounts Receivable'
@@ -39,7 +40,8 @@ payment_join as (
     select
         payments.payment_id as transaction_id,
         payments.source_relation,
-        row_number() over(partition by payments.payment_id order by payments.transaction_date) - 1 as index,
+        row_number() over(partition by payments.payment_id, payments.source_relation 
+            order by payments.source_relation, payments.transaction_date) - 1 as index,
         payments.transaction_date,
         payments.total_amount as amount,
         payments.deposit_to_account_id,
@@ -52,7 +54,7 @@ final as (
 
     select
         transaction_id,
-        source_relation,
+        payment_join.source_relation,
         index,
         transaction_date,
         customer_id,
@@ -69,7 +71,7 @@ final as (
 
     select
         transaction_id,
-        source_relation,
+        payment_join.source_relation,
         index,
         transaction_date,
         customer_id,
@@ -82,7 +84,8 @@ final as (
         'payment' as transaction_source
     from payment_join
 
-    cross join ar_accounts
+    inner join ar_accounts
+    on ar_accounts.source_relation = payment_join.source_relation
 )
 
 select *

models/double_entry_transactions/int_quickbooks__purchase_double_entry.sql

@@ -22,6 +22,7 @@ items as (
 
     left join {{ ref('stg_quickbooks__item') }} parent
         on item.parent_item_id = parent.item_id
+        and item.source_relation = parent.source_relation
 ),
 
 purchase_join as (

models/double_entry_transactions/int_quickbooks__refund_receipt_double_entry.sql

@@ -26,6 +26,7 @@ items as (
 
     left join {{ ref('stg_quickbooks__item') }} parent
         on item.parent_item_id = parent.item_id
+        and item.source_relation = parent.source_relation
 ),
 
 refund_receipt_join as (

models/double_entry_transactions/int_quickbooks__transfer_double_entry.sql

@@ -16,7 +16,8 @@ transfer_body as (
     select
         transfer_id as transaction_id,
         source_relation,
-        row_number() over(partition by transfer_id order by transaction_date) - 1 as index,
+        row_number() over(partition by transfer_id, source_relation 
+            order by source_relation, transaction_date) - 1 as index,
         transaction_date,
         amount,
         from_account_id as credit_to_account_id,

models/double_entry_transactions/int_quickbooks__vendor_credit_double_entry.sql

@@ -26,6 +26,7 @@ items as (
 
     left join {{ ref('stg_quickbooks__item') }} parent
         on item.parent_item_id = parent.item_id
+        and item.source_relation = parent.source_relation
 ),
 
 vendor_credit_join as (

models/intermediate/int_quickbooks__general_ledger_balances.sql

@@ -38,7 +38,8 @@ gl_cumulative_balance as (
     select
         *,
         case when financial_statement_helper = 'balance_sheet'
-            then sum(period_balance) over (partition by account_id, class_id order by date_month, account_id, class_id rows unbounded preceding) 
+            then sum(period_balance) over (partition by account_id, class_id, source_relation 
+            order by source_relation, date_month, account_id, class_id rows unbounded preceding) 
             else 0
                 end as cumulative_balance
     from gl_period_balance
@@ -104,6 +105,7 @@ gl_patch as (
 
     left join gl_beginning_balance
         on gl_beginning_balance.account_id = gl_accounting_periods.account_id
+            and gl_beginning_balance.source_relation = gl_accounting_periods.source_relation
             and gl_beginning_balance.date_month = gl_accounting_periods.period_first_day
             and gl_beginning_balance.date_year = gl_accounting_periods.date_year
             and coalesce(gl_beginning_balance.class_id, '0') = coalesce(gl_accounting_periods.class_id, '0')
@@ -116,7 +118,7 @@ gl_value_partition as (
         sum(case when period_ending_balance_starter is null 
             then 0 
             else 1 
-                end) over (order by account_id, class_id, period_last_day rows unbounded preceding) as gl_partition
+                end) over (order by source_relation, account_id, class_id, period_last_day rows unbounded preceding) as gl_partition
     from gl_patch
 ),
 
@@ -140,9 +142,11 @@ final as (
         period_last_day,
         coalesce(period_net_change,0) as period_net_change,
         coalesce(period_beginning_balance_starter,
-            first_value(period_ending_balance_starter) over (partition by gl_partition order by period_last_day rows unbounded preceding)) as period_beginning_balance,
+            first_value(period_ending_balance_starter) over (partition by gl_partition, source_relation 
+            order by source_relation, period_last_day rows unbounded preceding)) as period_beginning_balance,
         coalesce(period_ending_balance_starter,
-            first_value(period_ending_balance_starter) over (partition by gl_partition order by period_last_day rows unbounded preceding)) as period_ending_balance
+            first_value(period_ending_balance_starter) over (partition by gl_partition, source_relation 
+            order by source_relation, period_last_day rows unbounded preceding)) as period_ending_balance
     from gl_value_partition
 )
 

models/intermediate/int_quickbooks__retained_earnings.sql

@@ -66,7 +66,7 @@ retained_earnings_beginning as (
 
     select
         *,
-        sum(coalesce(period_net_change,0)) over (order by period_first_day, period_first_day rows unbounded preceding) as period_ending_balance
+        sum(coalesce(period_net_change,0)) over (order by source_relation, period_first_day, period_first_day rows unbounded preceding) as period_ending_balance
     from retained_earnings_starter
 ),
 
@@ -89,7 +89,7 @@ final as (
         period_first_day,
         period_last_day,
         period_net_change,
-        lag(coalesce(period_ending_balance,0)) over (order by period_first_day) as period_beginning_balance,
+        lag(coalesce(period_ending_balance,0)) over (order by source_relation, period_first_day) as period_beginning_balance,
         period_ending_balance
     from retained_earnings_beginning
 )