From 28ae8ca07d8dd231bab7cf51dc7c33604ad3a2c5 Mon Sep 17 00:00:00 2001 From: Jacob Matthews Date: Tue, 25 Jun 2024 16:00:54 +1000 Subject: [PATCH 1/3] Add ofx parser with support for bank australia --- go.mod | 6 ++ go.sum | 9 +++ internal/beanport/importer.go | 10 +++ internal/beanport/transactions.go | 8 -- internal/providers/amexcsv/importer.go | 7 +- internal/providers/ofx/importer.go | 106 +++++++++++++++++++++++++ main.go | 11 +++ 7 files changed, 148 insertions(+), 9 deletions(-) create mode 100644 internal/beanport/importer.go create mode 100644 internal/providers/ofx/importer.go diff --git a/go.mod b/go.mod index 8a34589..04b2beb 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,12 @@ go 1.22 require github.com/fatih/color v1.17.0 require ( + github.com/aclindsa/xml v0.0.0-20201125035057-bbd5c9ec99ac // indirect + golang.org/x/text v0.3.7 // indirect +) + +require ( + github.com/aclindsa/ofxgo v0.1.3 github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/go.sum b/go.sum index 0f15757..caa348a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/aclindsa/ofxgo v0.1.3 h1:20Ckjpg5gG4rdGh2juGfa5I1gnWULMXGWxpseVLCVaM= +github.com/aclindsa/ofxgo v0.1.3/go.mod h1:q2mYxGiJr5X3rlyoQjQq+qqHAQ8cTLntPOtY0Dq0pzE= +github.com/aclindsa/xml v0.0.0-20201125035057-bbd5c9ec99ac h1:xCNSfPWpcx3Sdz/+aB/Re4L8oA6Y4kRRRuTh1CHCDEw= +github.com/aclindsa/xml v0.0.0-20201125035057-bbd5c9ec99ac/go.mod h1:GjqOUT8xlg5+T19lFv6yAGNrtMKkZ839Gt4e16mBXlY= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -5,10 +9,15 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/beanport/importer.go b/internal/beanport/importer.go new file mode 100644 index 0000000..a99ba9d --- /dev/null +++ b/internal/beanport/importer.go @@ -0,0 +1,10 @@ +// The shared types for beanport +package beanport + +// An import provider +type Provider = string + +// The interface for a data importer +type Importer interface { + Import() ([]*PendingTransaction, error) +} diff --git a/internal/beanport/transactions.go b/internal/beanport/transactions.go index 3745568..798e33b 100644 --- a/internal/beanport/transactions.go +++ b/internal/beanport/transactions.go @@ -3,9 +3,6 @@ package beanport import "time" -// An import provider -type Provider = string - // A transaction that hasn't been confirmed type PendingTransaction struct { Index int @@ -22,8 +19,3 @@ type Transaction struct { PendingTransaction OppositeAccount string } - -// The interface for a data importer -type Importer interface { - Import() ([]*PendingTransaction, error) -} diff --git a/internal/providers/amexcsv/importer.go b/internal/providers/amexcsv/importer.go index 30b70ab..eb64c53 100644 --- a/internal/providers/amexcsv/importer.go +++ b/internal/providers/amexcsv/importer.go @@ -127,13 +127,18 @@ func (ai *amexImporter) Import() ([]*beanport.PendingTransaction, error) { parseErr = err break } + + if ai.config.NegativeAmounts { + amount = -amount + } + reference := split[13] txns = append(txns, &beanport.PendingTransaction{ Index: i, Account: ai.config.Account, Date: date, Description: description, - Amount: 0 - amount, + Amount: amount, Reference: reference, Commodity: ai.config.Commodity, }) diff --git a/internal/providers/ofx/importer.go b/internal/providers/ofx/importer.go new file mode 100644 index 0000000..4b9d0f1 --- /dev/null +++ b/internal/providers/ofx/importer.go @@ -0,0 +1,106 @@ +package ofx + +import ( + "bytes" + "crypto/md5" + "errors" + "fmt" + "strconv" + + "github.com/aclindsa/ofxgo" + "github.com/jacob-ian/beanport/internal/beanport" +) + +const ( + Provider beanport.Provider = "ofx" +) + +type ImporterConfig struct { + Commodity string + Account string +} + +type Importer struct { + data []byte + commodity string + account string +} + +func (imp *Importer) Import() ([]*beanport.PendingTransaction, error) { + res, err := ofxgo.ParseResponse(bytes.NewReader(imp.data)) + if err != nil { + return nil, errors.Join(errors.New("Couldn't parse OFX"), err) + } + + if len(res.Bank) != 1 { + return nil, errors.New("OFX error: too many bank messages") + } + + root, ok := res.Bank[0].(*ofxgo.StatementResponse) + if !ok { + return nil, errors.Join(errors.New("OFX error: not a statement response"), err) + } + + txns := root.BankTranList.Transactions + var output []*beanport.PendingTransaction + for i, txn := range txns { + var description string + if txn.Payee != nil { + description = txn.Payee.Name.String() + } else if txn.Name.String() != "" { + description = txn.Name.String() + } else if txn.Memo.String() != "" { + description = txn.Memo.String() + } else { + err = fmt.Errorf("No available description for FITID %v", txn.FiTID.String()) + break + } + + if description[:5] == "VISA-" { + description = description[5:] + } + + amt, err := strconv.ParseFloat(txn.TrnAmt.String(), 64) + if err != nil { + err = errors.Join(fmt.Errorf("Could not parse transaction amount for FITID %v: '%s'", txn.FiTID.String(), description), err) + break + } + + date := txn.DtPosted.Time + + var ref string + + if txn.CheckNum.String() != "" { + ref = txn.CheckNum.String() + } else if txn.RefNum.String() != "" { + ref = txn.RefNum.String() + } else { + input := fmt.Sprintf("%v:%s:%.2f", date.Format("2006-01-02"), description, amt) + ref = fmt.Sprintf("%x", md5.Sum([]byte(input))) + } + + output = append(output, &beanport.PendingTransaction{ + Index: i, + Account: imp.account, + Date: txn.DtPosted.Time, + Description: description, + Amount: amt, + Reference: ref, + Commodity: imp.commodity, + }) + } + + if err != nil { + return nil, errors.Join(errors.New("Could not read transactions"), err) + } + + return output, nil +} + +func NewImporter(data []byte, config *ImporterConfig) *Importer { + return &Importer{ + data: data, + commodity: config.Commodity, + account: config.Account, + } +} diff --git a/main.go b/main.go index 81e4926..d3771c2 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "github.com/jacob-ian/beanport/internal/beanport" "github.com/jacob-ian/beanport/internal/providers/amexcsv" + "github.com/jacob-ian/beanport/internal/providers/ofx" "github.com/jacob-ian/beanport/internal/tui" ) @@ -42,6 +43,16 @@ func main() { Commodity: cfg.Commodity, }) } + if cfg.Provider == ofx.Provider { + importer = ofx.NewImporter(data, &ofx.ImporterConfig{ + Account: cfg.Account, + Commodity: cfg.Commodity, + }) + } + + if importer == nil { + panic("Invalid provider selected") + } defaults, err := beanport.UserDefaultsFromFile(cfg.DefaultsFilePath) if err != nil { From 3601752cdbb07d3b0f3796d1c841169b261e348a Mon Sep 17 00:00:00 2001 From: Jacob Matthews Date: Wed, 26 Jun 2024 17:27:26 +1000 Subject: [PATCH 2/3] Improve ofx support for bank australia --- README.md | 7 ++-- internal/beanport/formatting.go | 3 +- internal/providers/ofx/description.go | 48 ++++++++++++++++++++++++ internal/providers/ofx/importer.go | 40 +++++++------------- internal/providers/ofx/reference.go | 54 +++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 31 deletions(-) create mode 100644 internal/providers/ofx/description.go create mode 100644 internal/providers/ofx/reference.go diff --git a/README.md b/README.md index d90e9e7..411baf0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,10 @@ A CLI tool to import transaction data into a beancount ledger, written in Go. ## Financial Institution Support -| Institution | Provider | -| ----------- | -------- | -| Amex (CSV) | amexcsv | +| Institution | Provider | +| -------------- | -------- | +| Amex (CSV) | amexcsv | +| Bank Australia | ofx | ## Get Started diff --git a/internal/beanport/formatting.go b/internal/beanport/formatting.go index 887390b..cfb1af6 100644 --- a/internal/beanport/formatting.go +++ b/internal/beanport/formatting.go @@ -23,7 +23,8 @@ func FormatTransaction(txn *Transaction) string { return fmt.Sprintf( "%s * \"%s REF:%s\"\n\t%s\t%.2f %s\n\t%s\t%.2f %s\n\n", txn.Date.Format("2006-01-02"), - txn.Description, txn.Reference, + txn.Description, + txn.Reference, txn.Account, txn.Amount, txn.Commodity, diff --git a/internal/providers/ofx/description.go b/internal/providers/ofx/description.go new file mode 100644 index 0000000..b117247 --- /dev/null +++ b/internal/providers/ofx/description.go @@ -0,0 +1,48 @@ +package ofx + +import ( + "fmt" + "strings" + + "github.com/aclindsa/ofxgo" +) + +// Gets a description from an OFX transaction +func description(txn ofxgo.Transaction) (string, error) { + if txn.Payee != nil { + return fmt.Sprintf("%s %s", txn.Payee.Name.String(), txn.Payee.State), nil + } + if txn.Name.String() != "" { + return txn.Name.String(), nil + } + if txn.Memo.String() != "" { + return descriptionFromMemo(txn.Memo.String()), nil + } + return "", fmt.Errorf("No available description properties for FITID %v", txn.FiTID.String()) +} + +// Convert a bank transaction memo to a description +func descriptionFromMemo(memo string) string { + description := memo + if description[:5] == "VISA-" { + description = description[5:] + } + + fields := strings.Fields(description) + if len(fields) == 0 { + return description + } + + var output []string + for _, f := range fields { + if strings.Contains(f, "Ref") { + continue + } + if strings.Contains(f, "Apple Pay") { + continue + } + output = append(output, f) + } + + return strings.Join(output, " ") +} diff --git a/internal/providers/ofx/importer.go b/internal/providers/ofx/importer.go index 4b9d0f1..44da59c 100644 --- a/internal/providers/ofx/importer.go +++ b/internal/providers/ofx/importer.go @@ -2,10 +2,11 @@ package ofx import ( "bytes" - "crypto/md5" "errors" "fmt" + "slices" "strconv" + "strings" "github.com/aclindsa/ofxgo" "github.com/jacob-ian/beanport/internal/beanport" @@ -43,42 +44,27 @@ func (imp *Importer) Import() ([]*beanport.PendingTransaction, error) { txns := root.BankTranList.Transactions var output []*beanport.PendingTransaction + var parseErr error + for i, txn := range txns { - var description string - if txn.Payee != nil { - description = txn.Payee.Name.String() - } else if txn.Name.String() != "" { - description = txn.Name.String() - } else if txn.Memo.String() != "" { - description = txn.Memo.String() - } else { - err = fmt.Errorf("No available description for FITID %v", txn.FiTID.String()) + description, err := description(txn) + if err != nil { + parseErr = errors.Join(fmt.Errorf("Could not generate description"), err) break } - if description[:5] == "VISA-" { - description = description[5:] - } - amt, err := strconv.ParseFloat(txn.TrnAmt.String(), 64) if err != nil { - err = errors.Join(fmt.Errorf("Could not parse transaction amount for FITID %v: '%s'", txn.FiTID.String(), description), err) + parseErr = errors.Join(fmt.Errorf("Could not parse transaction amount for FITID %v: '%s'", txn.FiTID.String(), description), err) break } - date := txn.DtPosted.Time - - var ref string - - if txn.CheckNum.String() != "" { - ref = txn.CheckNum.String() - } else if txn.RefNum.String() != "" { - ref = txn.RefNum.String() - } else { - input := fmt.Sprintf("%v:%s:%.2f", date.Format("2006-01-02"), description, amt) - ref = fmt.Sprintf("%x", md5.Sum([]byte(input))) + if amt == 0.00 { + continue } + ref := reference(txn, description, amt) + output = append(output, &beanport.PendingTransaction{ Index: i, Account: imp.account, @@ -90,7 +76,7 @@ func (imp *Importer) Import() ([]*beanport.PendingTransaction, error) { }) } - if err != nil { + if parseErr != nil { return nil, errors.Join(errors.New("Could not read transactions"), err) } diff --git a/internal/providers/ofx/reference.go b/internal/providers/ofx/reference.go new file mode 100644 index 0000000..ae02b75 --- /dev/null +++ b/internal/providers/ofx/reference.go @@ -0,0 +1,54 @@ +package ofx + +import ( + "crypto/md5" + "encoding/base64" + "fmt" + "strings" + + "github.com/aclindsa/ofxgo" +) + +// Gets a reference for an OFX transaction +// The reference is designed to prevent ledger duplicates +func reference(txn ofxgo.Transaction, description string, amount float64) string { + if txn.CheckNum.String() != "" { + return txn.CheckNum.String() + } + if txn.RefNum.String() != "" { + return txn.RefNum.String() + } + + if ref := referenceFromMemo(txn.Memo.String()); ref != "" { + return ref + } + + input := fmt.Sprintf("%v:%s:%.2f", txn.DtPosted.Time.Format("2006-01-02"), description, amount) + hashed := fmt.Sprintf("%x", md5.Sum([]byte(input))) + encoded := base64.StdEncoding.EncodeToString([]byte(hashed)) + return encoded +} + +// Attempts to find a ref in the transaction memo +func referenceFromMemo(memo string) string { + if memo == "" { + return "" + } + + fields := strings.Fields(memo) + if len(fields) == 0 { + return "" + } + + reference := "" + for _, f := range fields { + exists := strings.Contains(f, "Ref") + if !exists { + continue + } + reference = f + break + } + + return reference +} From f99d12f9ece853037f31fa78341643374f91385b Mon Sep 17 00:00:00 2001 From: Jacob Matthews Date: Thu, 27 Jun 2024 16:38:23 +1000 Subject: [PATCH 3/3] Remove Apple Pay from description --- internal/providers/ofx/description.go | 10 ++++++++-- internal/providers/ofx/importer.go | 2 -- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/providers/ofx/description.go b/internal/providers/ofx/description.go index b117247..6090cea 100644 --- a/internal/providers/ofx/description.go +++ b/internal/providers/ofx/description.go @@ -34,13 +34,19 @@ func descriptionFromMemo(memo string) string { } var output []string - for _, f := range fields { + for i, f := range fields { if strings.Contains(f, "Ref") { continue } - if strings.Contains(f, "Apple Pay") { + + if i > 0 && fields[i-1] == "Apple" && f == "Pay" { continue } + + if i < len(fields)-1 && f == "Apple" && fields[i+1] == "Pay" { + continue + } + output = append(output, f) } diff --git a/internal/providers/ofx/importer.go b/internal/providers/ofx/importer.go index 44da59c..95a95ac 100644 --- a/internal/providers/ofx/importer.go +++ b/internal/providers/ofx/importer.go @@ -4,9 +4,7 @@ import ( "bytes" "errors" "fmt" - "slices" "strconv" - "strings" "github.com/aclindsa/ofxgo" "github.com/jacob-ian/beanport/internal/beanport"