Merge pull request #3 from jbettenh/develop

Develop

.gitignore

@@ -29,3 +29,5 @@
 # File inputs
 
 # File outputs
+/cleaned_files/**
+

fbarhelper/models.py

@@ -0,0 +1,58 @@
+import sqlite3
+
+
+def drop_tables():
+    conn = sqlite3.connect('fbar.db')
+
+    with conn:
+        cursor = conn.cursor()
+        cursor.execute("DROP TABLE TRANSLATIONS")
+        cursor.execute("DROP TABLE TRANSACTIONS")
+
+
+def insert_tables():
+    conn = sqlite3.connect('fbar.db')
+
+    with conn:
+        cursor = conn.cursor()
+        cursor.execute("""CREATE TABLE IF NOT EXISTS
+               TRANSLATIONS(ID INTEGER PRIMARY KEY, DE TEXT, EN TEXT, DESCRIPTION TEXT)""")
+
+        cursor.execute("""CREATE TABLE IF NOT EXISTS
+                   TRANSACTIONS(ID INTEGER PRIMARY KEY not null on conflict ignore, BOOKING_DATE DATE, DATE DATE, 
+                   TRANSACTION_TYPE TEXT, RECIPIENT TEXT, USAGE TEXT, IBAN TEXT, BIC TEXT, CUSTOMER_REF TEXT, 
+                   MANDATE_REF TEXT,CREDITOR_ID TEXT, FOREIGN_FEES TEXT, SUM REAL, ALTERNATIVE_RECIPIENT TEXT, 
+                   NO_ORDERS TEXT, NO_CHECKS TEXT, DEBIT REAL, CREDIT REAL, CURRENCY TEXT)""")
+
+
+def translate_column_names():
+    conn = sqlite3.connect('fbar.db')
+
+    with conn:
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('0', 'Buchungstag', 'Booking Date',"
+                     "'Date Deutsche Bank recorded the transaction' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('1', 'Wert', 'Date',"
+                     "'Date Deutsche Bank recorded the transaction' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('2', 'Umsatzart', 'Transaction Type', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('3', 'Begünstigter / Auftraggaber', 'Recipient', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('4', 'Verwendungszweck', 'Usage','Intended purpose memo' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('5', 'Kundenreferenz', 'Customer reference', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('6', 'Mandatsreferenz', 'Mandate reference', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('7', 'Gläubiger ID', 'Creditor identifier', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('8', 'Fremde Gebühren', 'Foreign fees', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('9', 'Betrag', 'Sum', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('10', 'Abweichender Empfänger', 'Alternative Recipient', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('11', 'Anzahl der Aufträge', 'No of orders', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('12', 'Anzahl der Schecks', 'No of checks', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('13', 'Soll', 'Debit', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('14', 'Haben', 'Credit', '' )")
+        conn.execute("INSERT INTO TRANSLATIONS VALUES ('15', 'Währung', 'Currency', '' )")
+
+
+if __name__ == '__main__':
+    drop_tables()
+    print('Tables dropped.')
+    insert_tables()
+    print('Tables inserted.')
+    translate_column_names()
+    print('Inserted header translations.')

fbarhelper/transform.py

@@ -1,33 +1,100 @@
 import csv
+import os
 import sqlite3
 
 import pandas
+import pandas as pd
+
+
+def main(indir, outdir):
+
+    li = []
+
+    for root, dirs, filenames in os.walk(indir):
+        for f in filenames:
+            raw_filename = os.path.join(root, f)
+            cleaned_filename = os.path.join(outdir, 'cleaned_' + f)
+            if '.csv' in raw_filename:
+                csv_cleaner(raw_filename, cleaned_filename)
+
+    for root, dirs, filenames in os.walk(outdir):
+        for filename in filenames:
+            filepath = os.path.join(root, filename)
+            df = import_bank_csv(filepath)
+            li.append(df)
+
+    frame = pd.concat(li, axis=0, ignore_index=True)
+    import_bank_data_to_db(frame)
+
+
+def csv_cleaner(raw_file, cleaned_file, header_rows=3, footer_rows=1, ):
+    """
+    Take the raw csv export from Deutsche Bank and create a csv of just transactions.
+    """
+    print(f'Cleaning CSV files in {raw_file} .....')
+    try:
+        with open(raw_file, 'r', encoding='ISO-8859-1') as fin, \
+                open(cleaned_file, 'w', encoding='ISO-8859-1', newline='') as fout:
+
+            file = csv.reader(fin, delimiter=';', lineterminator='\n')
+            writer = csv.writer(fout, delimiter=';')
+
+            for current_row, line in enumerate(file):
+                if current_row > header_rows and "Account balance" not in line:
+                    writer.writerow(line)
+
+    except IOError as err:
+        print(f"IOError {err} in ", raw_file)
 
 
 def import_bank_csv(csv_file):
 
+    print(f'Importing bank csv {csv_file} .....')
     df = pandas.read_csv(csv_file,
                          sep=';',
                          encoding='ISO-8859-1',
                          quoting=csv.QUOTE_NONE
                          )
-    data = df.replace('"', '', regex=True)
-
-    # Convert from dd.mm.yyyy to yyyy-mm-dd
-    data['Buchungstag'] = pandas.to_datetime(data['Buchungstag'], format='%d.%m.%Y').dt.date
-    data['Wert'] = pandas.to_datetime(data['Wert'], format='%d.%m.%Y').dt.date
+    df = df.replace('"', '', regex=True)
+    df.columns = ['BOOKING_DATE', 'DATE', 'TRANSACTION_TYPE', 'RECIPIENT', 'USAGE', 'IBAN', 'BIC', 'CUSTOMER_REF',
+                  'MANDATE_REF', 'CREDITOR_ID', 'FOREIGN_FEES', 'SUM', 'ALTERNATIVE_RECIPIENT', 'NO_ORDERS',
+                  'NO_CHECKS', 'DEBIT', 'CREDIT', 'CURRENCY']
+
+    df['BOOKING_DATE'] = pandas.to_datetime(df['BOOKING_DATE'], format='%m/%d/%Y').dt.date
+    df['DATE'] = pandas.to_datetime(df['DATE'], format='%m/%d/%Y').dt.date
 
-    return data
+    return df
 
 
 def import_bank_data_to_db(bank_data):
-    bank_data.to_sql(name='transactions',
+    print('Adding to database .....')
+    bank_data.to_sql(name='TRANSACTIONS',
                      con=sqlite3.connect('fbar.db'),
-                     if_exists='replace',
-                     index=True)
+                     if_exists='append',
+                     index=True,
+                     index_label='ID')
+
+
+def get_max_credit():
+    conn = sqlite3.connect('fbar.db')
+
+    with conn:
+        c = conn.cursor()
+        c.execute("SELECT MAX(CREDIT), BOOKING_DATE FROM TRANSACTIONS;")
+        print(f'Largest credit was: {c.fetchone()}')
+
+
+def get_max_debit():
+    conn = sqlite3.connect('fbar.db')
+
+    with conn:
+        c = conn.cursor()
+        c.execute("SELECT MIN(DEBIT), BOOKING_DATE FROM TRANSACTIONS;")
+        print(f'Largest debit was: {c.fetchone()}')
 
 
 if __name__ == '__main__':
-    db = import_bank_csv('C:/code/python3/fbarhelper/tests/testdata/Transactions_999_123456789_20210622_092547.csv')
-    import_bank_data_to_db(db)
+    main('C:/code/python3/fbarhelper/tests/testdata/', 'C:/code/python3/fbarhelper/cleaned_files/')
+    get_max_credit()
+    get_max_debit()
 

pyproject.toml

@@ -19,4 +19,4 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
-transform = 'fbarhelper.transform:main'
+transform = "fbarhelper.transform:main"