Merge pull request #36 from martinohansen/martin/new-import-ids

martin/new-imports-ids

cmd/ynabber/main.go

@@ -29,13 +29,19 @@ func main() {
 		log.Fatal("YNAB_CLEARED must be one of cleared, uncleared or reconciled")
 	}
 
-	// Handle movement of PayeeStrip from YNAB to Nordigen config strut
+	// Handle movement of config options and warn users
 	if cfg.Nordigen.PayeeStrip == nil {
 		if cfg.PayeeStrip != nil {
 			log.Printf("Config YNABBER_PAYEE_STRIP is depreciated, please use NORDIGEN_PAYEE_STRIP instead")
 			cfg.Nordigen.PayeeStrip = cfg.PayeeStrip
 		}
 	}
+	if cfg.YNAB.AccountMap == nil {
+		if cfg.Nordigen.AccountMap != nil {
+			log.Printf("Config NORDIGEN_ACCOUNTMAP is depreciated, please use YNAB_ACCOUNTMAP instead")
+			cfg.YNAB.AccountMap = cfg.Nordigen.AccountMap
+		}
+	}
 
 	if cfg.Debug {
 		log.Printf("Config: %+v\n", cfg)

config.go

@@ -59,8 +59,7 @@ type Config struct {
 
 // Nordigen related settings
 type Nordigen struct {
-	// AccountMap of Nordigen account IDs to YNAB account IDs in JSON. For
-	// example: '{"<nordigen account id>": "<ynab account id>"}'
+	// AccountMap is depreciated please use YNAB.AccountMap instead
 	AccountMap AccountMap `envconfig:"NORDIGEN_ACCOUNTMAP"`
 
 	// BankID is used to create requisition
@@ -99,9 +98,33 @@ type YNAB struct {
 	// settings section
 	Token string `envconfig:"YNAB_TOKEN"`
 
+	// AccountMap of IBAN to YNAB account IDs in JSON. For example:
+	// '{"<IBAN>": "<YNAB Account ID>"}'
+	AccountMap AccountMap `envconfig:"YNAB_ACCOUNTMAP"`
+
+	// FromDate only import transactions from this date and onward. For
+	// example: 2006-01-02
+	FromDate Date `envconfig:"YNAB_FROM_DATE"`
+
 	// Set cleared status, possible values: cleared, uncleared, reconciled .
 	// Default is uncleared for historical reasons but recommend setting this
 	// to cleared because ynabber transactions are cleared by bank.
 	// They'd still be unapproved until approved in YNAB.
 	Cleared string `envconfig:"YNAB_CLEARED" default:"uncleared"`
+
+	ImportID ImportID
+}
+
+// ImportID can be either v1 or v2. All new users should use v2 because it
+// have a lower potability of making duplicate transactions. But v1 remains
+// the default to retain backwards compatibility.
+//
+// To migrate from v1 to v2 simply set the v2 to any date and all transactions
+// from and including that date will be using v2 of the import ID generator.
+type ImportID struct {
+	// V1 will be used from this date
+	V1 Date `envconfig:"YNAB_IMPORT_ID_V1" default:"1970-01-01"`
+
+	// V2 will be used from this date, for example: 2022-12-24
+	V2 Date `envconfig:"YNAB_IMPORT_ID_V2" default:"9999-01-01"`
 }

config_test.go

@@ -0,0 +1,37 @@
+package ynabber
+
+import (
+	"testing"
+	"time"
+)
+
+func TestDateDecode(t *testing.T) {
+	type args struct {
+		value string
+	}
+	tests := []struct {
+		name    string
+		date    *Date
+		args    args
+		want    time.Time
+		wantErr bool
+	}{
+		{
+			date:    &Date{},
+			args:    args{value: "2000-12-24"},
+			want:    time.Date(2000, 12, 24, 0, 0, 0, 0, time.UTC),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := &Date{}
+			if err := got.Decode(tt.args.value); (err != nil) != tt.wantErr {
+				t.Errorf("Date.Decode() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if time.Time(*got) != tt.want {
+				t.Errorf("Date.Decode() got = %v, want %v", time.Time(*got), tt.want)
+			}
+		})
+	}
+}

reader/nordigen/nordigen.go

@@ -17,21 +17,6 @@ import (
 
 const timeLayout = "2006-01-02"
 
-// TODO(Martin): Move accountParser from Nordigen to YNAB package. We want to
-// map Ynabber transaction to YNAB and not so much Nordigen to Ynabber like this
-// is during currently.
-func accountParser(account string, accountMap map[string]string) (ynabber.Account, error) {
-	for from, to := range accountMap {
-		if account == from {
-			return ynabber.Account{
-				ID:   ynabber.ID(to),
-				Name: from,
-			}, nil
-		}
-	}
-	return ynabber.Account{}, fmt.Errorf("account not found in map: %w", ynabber.ErrNotFound)
-}
-
 // payeeStrip returns payee with elements of strips removed
 func payeeStrip(payee string, strips []string) (x string) {
 	x = payee
@@ -176,21 +161,16 @@ func BulkReader(cfg ynabber.Config) (t []ynabber.Transaction, err error) {
 			)
 			Authorization.CreateAndSave()
 		}
-		accountID := accountMetadata.Id
-		accountName := accountMetadata.Iban
 
-		account, err := accountParser(accountName, cfg.Nordigen.AccountMap)
-		if err != nil {
-			if errors.Is(err, ynabber.ErrNotFound) {
-				log.Printf("No matching account found for: %s in: %v", accountName, cfg.Nordigen.AccountMap)
-				continue
-			}
-			return nil, err
+		account := ynabber.Account{
+			ID:   ynabber.ID(accountMetadata.Id),
+			Name: accountMetadata.Iban,
+			IBAN: accountMetadata.Iban,
 		}
 
-		log.Printf("Reading transactions from account: %s", accountName)
+		log.Printf("Reading transactions from account: %s", account.Name)
 
-		transactions, err := c.GetAccountTransactions(accountID)
+		transactions, err := c.GetAccountTransactions(string(account.ID))
 		if err != nil {
 			return nil, fmt.Errorf("failed to get transactions: %w", err)
 		}

reader/nordigen/nordigen_test.go

@@ -65,37 +65,6 @@ func TestTransactionToYnabber(t *testing.T) {
 	}
 }
 
-func TestAccountParser(t *testing.T) {
-	type args struct {
-		account    string
-		accountMap map[string]string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    ynabber.Account
-		wantErr bool
-	}{
-		{name: "match",
-			args:    args{account: "N1", accountMap: map[string]string{"N1": "Y1"}},
-			want:    ynabber.Account{Name: "Y1"},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := accountParser(tt.args.account, tt.args.accountMap)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if got.Name != tt.args.account {
-				t.Errorf("got = %v, want %v", got.Name, tt.args.account)
-			}
-		})
-	}
-}
-
 func TestPayeeStripNonAlphanumeric(t *testing.T) {
 	want := "Im just alphanumeric"
 	got := payeeStripNonAlphanumeric("Im just alphanumeric")

writer/ynab/ynab.go

@@ -10,6 +10,7 @@ import (
 	"net/http/httputil"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/martinohansen/ynabber"
 )
@@ -36,7 +37,62 @@ type Ytransactions struct {
 	Transactions []Ytransaction `json:"transactions"`
 }
 
-func ynabberToYNAB(cfg ynabber.Config, t ynabber.Transaction) Ytransaction {
+// accountParser takes IBAN and returns the matching YNAB account ID in
+// accountMap
+func accountParser(iban string, accountMap map[string]string) (string, error) {
+	for from, to := range accountMap {
+		if iban == from {
+			return to, nil
+		}
+	}
+	return "", fmt.Errorf("no account for: %s in map: %s", iban, accountMap)
+}
+
+// importIDMaker tries to return a unique YNAB import ID to avoid duplicate
+// transactions.
+func importIDMaker(cfg ynabber.Config, t ynabber.Transaction) string {
+	// Common between versions
+	date := t.Date.Format("2006-01-02")
+	amount := t.Amount.String()
+
+	// Version 1 uses the memo, amount and date from Ytransaction
+	v1Cutover := time.Time(cfg.YNAB.ImportID.V1)
+	v1 := func(t ynabber.Transaction) string {
+		hash := sha256.Sum256([]byte(t.Memo))
+		return fmt.Sprintf("YBBR:%s:%s:%x", amount, date, hash[:2])
+	}
+
+	// Version 2 uses, in order, the account IBAN, transaction ID, date, and
+	// amount to build a hash of the transaction.
+	v2Cutover := time.Time(cfg.YNAB.ImportID.V2)
+	v2 := func(t ynabber.Transaction) string {
+		s := [][]byte{
+			[]byte(t.Account.IBAN),
+			[]byte(t.ID),
+			[]byte(date),
+			[]byte(amount),
+		}
+		hash := sha256.Sum256(bytes.Join(s, []byte("")))
+		return fmt.Sprintf("YBBR:%x", hash)[:32]
+	}
+
+	// Return the first generator from latest to oldest that have a cutover date
+	// after or equal to the transaction date.
+	if t.Date.After(v2Cutover) || t.Date.Equal(v2Cutover) {
+		return v2(t)
+	} else if t.Date.After(v1Cutover) || t.Date.Equal(v1Cutover) {
+		return v1(t)
+	} else {
+		return v1(t)
+	}
+}
+
+func ynabberToYNAB(cfg ynabber.Config, t ynabber.Transaction) (Ytransaction, error) {
+	accountID, err := accountParser(t.Account.IBAN, cfg.YNAB.AccountMap)
+	if err != nil {
+		return Ytransaction{}, err
+	}
+
 	date := t.Date.Format("2006-01-02")
 	amount := t.Amount.String()
 
@@ -56,33 +112,46 @@ func ynabberToYNAB(cfg ynabber.Config, t ynabber.Transaction) Ytransaction {
 		payee = payee[0:(maxPayeeSize - 1)]
 	}
 
-	// Generating YNAB compliant import ID, output example:
-	// YBBR:-741000:2021-02-18:92f2beb1
-	hash := sha256.Sum256([]byte(t.Memo))
-	id := fmt.Sprintf("YBBR:%s:%s:%x", amount, date, hash[:2])
-
 	return Ytransaction{
-		AccountID: string(t.Account.ID),
+		ImportID:  importIDMaker(cfg, t),
+		AccountID: accountID,
 		Date:      date,
 		Amount:    amount,
 		PayeeName: payee,
 		Memo:      memo,
-		ImportID:  id,
 		Cleared:   cfg.YNAB.Cleared,
 		Approved:  false,
-	}
+	}, nil
 }
 
 func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error {
-	if len(t) == 0 {
-		log.Println("No transactions to write")
-		return nil
-	}
+	// skipped and failed counters
+	skipped := 0
+	failed := 0
 
 	// Build array of transactions to send to YNAB
 	y := new(Ytransactions)
 	for _, v := range t {
-		y.Transactions = append(y.Transactions, ynabberToYNAB(cfg, v))
+		// Skip transaction if the date is before FromDate
+		if v.Date.Before(time.Time(cfg.YNAB.FromDate)) {
+			skipped += 1
+			continue
+		}
+
+		transaction, err := ynabberToYNAB(cfg, v)
+		if err != nil {
+			// If we fail to parse a single transaction we log it but move on so
+			// we don't halt the entire program.
+			log.Printf("Failed to parse transaction: %s: %s", v, err)
+			failed += 1
+			continue
+		}
+		y.Transactions = append(y.Transactions, transaction)
+	}
+
+	if len(t) == 0 || len(y.Transactions) == 0 {
+		log.Println("No transactions to write")
+		return nil
 	}
 
 	url := fmt.Sprintf("https://api.youneedabudget.com/v1/budgets/%s/transactions", cfg.YNAB.BudgetID)
@@ -119,7 +188,12 @@ func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error {
 	if res.StatusCode != http.StatusCreated {
 		return fmt.Errorf("failed to send request: %s", res.Status)
 	} else {
-		log.Printf("Successfully sent %v transaction(s) to YNAB", len(y.Transactions))
+		log.Printf(
+			"Successfully sent %v transaction(s) to YNAB. %d got skipped and %d failed.",
+			len(y.Transactions),
+			skipped,
+			failed,
+		)
 	}
 	return nil
 }