From 1a731bcb8624525294113d99095a00c2b45e6e9e Mon Sep 17 00:00:00 2001
From: Kartik Ohri <kartikohri13@gmail.com>
Date: Wed, 11 Sep 2024 19:45:20 +0530
Subject: [PATCH 01/11] Prefill donation form if user is logged in

---
 metabrainz/payments/views.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/metabrainz/payments/views.py b/metabrainz/payments/views.py
index bdc9f15c..cf3000a7 100644
--- a/metabrainz/payments/views.py
+++ b/metabrainz/payments/views.py
@@ -1,6 +1,7 @@
 from __future__ import division
 from flask import Blueprint, request, render_template, url_for, redirect, current_app, jsonify
 from flask_babel import gettext
+from flask_login import current_user
 from werkzeug.datastructures import MultiDict
 
 from metabrainz.payments import SUPPORTED_CURRENCIES
@@ -20,7 +21,10 @@
 @payments_bp.route('/donate')
 def donate():
     """Regular donation page."""
-    return render_template('payments/donate.html', form=DonationForm())
+    form = DonationForm()
+    if current_user is not None and not current_user.is_anonymous:
+        form.editor.data = current_user.musicbrainz_id
+    return render_template('payments/donate.html', form=form)
 
 
 @payments_bp.route('/payment/')

From d55b3e478d7ffae8e685253f8d8f501371deae16 Mon Sep 17 00:00:00 2001
From: Kartik Ohri <kartikohri13@gmail.com>
Date: Sat, 14 Sep 2024 01:14:26 +0530
Subject: [PATCH 02/11] Add prefill for donation amount, currency and editor

---
 metabrainz/payments/views.py              | 34 +++++++++++++++++++++--
 metabrainz/templates/payments/donate.html |  2 +-
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/metabrainz/payments/views.py b/metabrainz/payments/views.py
index cf3000a7..b8105807 100644
--- a/metabrainz/payments/views.py
+++ b/metabrainz/payments/views.py
@@ -1,10 +1,13 @@
 from __future__ import division
+
+from decimal import Decimal, InvalidOperation
+
 from flask import Blueprint, request, render_template, url_for, redirect, current_app, jsonify
 from flask_babel import gettext
 from flask_login import current_user
 from werkzeug.datastructures import MultiDict
 
-from metabrainz.payments import SUPPORTED_CURRENCIES
+from metabrainz.payments import SUPPORTED_CURRENCIES, Currency
 from metabrainz.model.payment import Payment
 from metabrainz.payments.forms import DonationForm, PaymentForm
 from metabrainz import flash
@@ -22,8 +25,33 @@
 def donate():
     """Regular donation page."""
     form = DonationForm()
-    if current_user is not None and not current_user.is_anonymous:
-        form.editor.data = current_user.musicbrainz_id
+
+    if editor := request.args.get('editor'):
+        form.editor.data = editor
+    else:
+        if current_user is not None and not current_user.is_anonymous:
+            form.editor.data = current_user.musicbrainz_id
+
+    amount = None
+    if _amount := request.args.get('amount'):
+        try:
+            value = Decimal(_amount)
+            if value >= 0:
+                amount = value
+        except (ValueError, TypeError, InvalidOperation):
+            pass
+
+    if amount is None:
+        amount = Decimal(50)
+    form.amount.data = amount
+
+    _currency =  request.args.get('currency', 'usd')
+    if _currency.lower() == 'eur':
+        currency = 'eur'
+    else:
+        currency = 'usd'
+    form.currency.data = currency
+
     return render_template('payments/donate.html', form=form)
 
 
diff --git a/metabrainz/templates/payments/donate.html b/metabrainz/templates/payments/donate.html
index a15675d8..feae52b8 100644
--- a/metabrainz/templates/payments/donate.html
+++ b/metabrainz/templates/payments/donate.html
@@ -39,7 +39,7 @@ <h1 class="page-title">{{ _('Make a Donation') }}</h1>
           <label for="input-amount" class="col-sm-offset-2 col-sm-3 control-label">{{ _('Amount') }}</label>
           <div class="col-sm-4">
             <div id="input-amount-fg">
-              {{ form.amount(id="input-amount", class="form-control", value="50.00") | safe }}
+              {{ form.amount(id="input-amount", class="form-control") | safe }}
               <div id="currency-selector" class="btn-group" data-toggle="buttons">
                 {% for subfield in form.currency %}
                   <label class="btn btn-primary {{ 'active' if subfield.checked }}">

From 914a908e923290b57ec6aa44fe566c8e9fde07f0 Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Tue, 24 Sep 2024 20:22:00 +0530
Subject: [PATCH 03/11] Start drastically improving the import script

---
 misc/parse-paypal-us.py | 52 +++++++++++++----------------------------
 1 file changed, 16 insertions(+), 36 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index 5b40872d..94a8ac86 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -11,7 +11,7 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
         yield [cell for cell in row]
 
 if len(sys.argv) not in [3,4]:
-    print("Usage parse-paypal-es.py <paypal csv file> <qbo csv file> [start balance]")
+    print("Usage parse-paypal-es.py <paypal csv file> <qbo csv file>")
     sys.exit(-1)
 
 fp = None
@@ -28,13 +28,6 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
     print("Cannot open output file %s" % sys.argv[2])
     sys.exit(0)
 
-balance = None
-try:
-    balance = Decimal(sys.argv[3])
-    print("got starting balance", balance)
-except (InvalidOperation, IndexError):
-    print("Cannot balance value, ignoring.")
-
 
 out = csv.writer(_out, quoting=csv.QUOTE_MINIMAL)
 out.writerow(["Date","Description","Amount"])
@@ -52,10 +45,9 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
 
     lines.append(line)
 
-index = 1
+index = 0
 register = []
-credit = Decimal(0.0)
-debits = Decimal(0.0)
+balance = None
 while True:
     if index >= len(lines):
         break
@@ -77,19 +69,14 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
         amount = gross
 
     elif currency != 'USD':
-#        print("transaction foreign")
         # Received money in foreign currency
-#        print(lines[index + 1])
-#        print(lines[index + 2])
         foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
         native = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
 
-
-#        print("native %f, foreign %f" % (native, foreign))
+        # Get the correct balance, because WTF paypal.
+        pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
 
         ratio = native / foreign
-#        print("conversion rate: %f" % ratio)
-
         fee = Decimal(fee) / ratio
         fee = Decimal(int(fee * 100)) / 100
         net = native - fee
@@ -100,17 +87,17 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
 
         index += 2
 
+    # If this is the first row, calculate the starting balance
+    if balance is None:
+        balance = pp_balance - net
+        print("Starting balance: ", balance)
+
     desc = desc.replace(",", " ")
     out.writerow([dat, desc, amount])
 
-    if amount > 0.0:
-        credit += amount
-    else:
-        debits += amount
-
-    if balance is not None:
-        balance = balance + Decimal(str(net).replace(",", ""))
-        register.append("%s %-40s %10s %10s %10s" % (dat, desc, str(amount), str(fee), str(pp_balance)))
+    balance = balance + net
+    print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(amount), str(fee),
+                                           str(pp_balance), str(balance)))
 
     desc = "PayPal Fee"
     dat = fields[0]
@@ -119,16 +106,9 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
 
     index += 1
 
+print("    Final paypal balance: ", pp_balance)
+print("Final calculated balance: ", balance)
+
 
 fp.close()
 _out.close()
-
-if register:
-    register.reverse()
-    for line in register:
-        print(line)
-
-    print()
-    print(" Debits: %.2f" % debits)
-    print("Credits: %.2f" % credit)
-    print("  Total: %.2f" % (credit + debits))

From 1886d9e04711b6e711ba2c3dad62885503c69dc0 Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Tue, 24 Sep 2024 21:37:21 +0530
Subject: [PATCH 04/11] Make currency conversion more robust

---
 misc/parse-paypal-us.py | 48 ++++++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 13 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index 94a8ac86..069c6a92 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -45,6 +45,8 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
 
     lines.append(line)
 
+print("DATE       NAME                                           GROSS       FEE     PP_BAL    BALANCE")
+
 index = 0
 register = []
 balance = None
@@ -63,40 +65,60 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
     status = fields[5]
     typ = fields[4]
 
+    if gross == Decimal(0.0):
+        print("*** skip non balance affecting transaction: %s", desc)
+        index += 1
+        continue
+
     currency = fields[6]
     if currency == 'USD' and typ != "General Currency Conversion":
         # Normal native currency transactions
         amount = gross
 
-    elif currency != 'USD':
+    elif currency != 'USD' and lines[index + 1][4] == "General Currency Conversion":
         # Received money in foreign currency
         foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
-        native = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
+        usd = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
+        print("      foreign: ", foreign)
+        print("  foreign fee: ", fee)
 
         # Get the correct balance, because WTF paypal.
         pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
 
-        ratio = native / foreign
-        fee = Decimal(fee) / ratio
-        fee = Decimal(int(fee * 100)) / 100
-        net = native - fee
+        exchange_rate = usd / foreign
+        usd_fee = Decimal(fee) * exchange_rate
+        usd_fee = Decimal(int(usd_fee * 100)) / 100
+        gross = usd - usd_fee
+        print("          net: ", usd)
+        print("      usd fee: ", usd_fee)
+        print("        gross: ", gross)
+        print("exchange rate: ", exchange_rate)
+        print("          net: ", net)
 
-        amount = native
         if typ == "Express Checkout Payment":
-            amount = -amount
+            gross = -gross
+
+        fee = usd_fee
+        net = foreign
+
 
         index += 2
+    else:
+        print("*** skip non balance affecting transaction: %s", desc)
+        index += 1
+        continue
 
-    # If this is the first row, calculate the starting balance
     if balance is None:
         balance = pp_balance - net
-        print("Starting balance: ", balance)
+
+    balance = balance + net
+    if balance != pp_balance:
+        print("     discrepancy: ", (pp_balance - balance))
 
     desc = desc.replace(",", " ")
-    out.writerow([dat, desc, amount])
+    out.writerow([dat, desc, fee, gross])
 
-    balance = balance + net
-    print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(amount), str(fee),
+    print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
                                            str(pp_balance), str(balance)))
 
     desc = "PayPal Fee"

From 07338ea68e0e4daa7bbb55681bbc5ef35a13364d Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Tue, 24 Sep 2024 22:29:10 +0530
Subject: [PATCH 05/11] Making progress

---
 misc/parse-paypal-us.py | 59 +++++++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 17 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index 069c6a92..acf355e5 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -65,48 +65,70 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
     status = fields[5]
     typ = fields[4]
 
-    if gross == Decimal(0.0):
-        print("*** skip non balance affecting transaction: %s", desc)
-        index += 1
-        continue
+#    if gross == Decimal(0.0):
+#        print("*** skip non balance affecting transaction: %s", desc)
+#        index += 1
+#        continue
 
     currency = fields[6]
+
+    try:
+        lookahead_type = lines[index + 1][4]
+    except IndexError:
+        lookahead_type = ""
+
     if currency == 'USD' and typ != "General Currency Conversion":
         # Normal native currency transactions
         amount = gross
 
-    elif currency != 'USD' and lines[index + 1][4] == "General Currency Conversion":
-        # Received money in foreign currency
-        foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
-        usd = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
-        print("      foreign: ", foreign)
-        print("  foreign fee: ", fee)
+    elif currency != 'USD' and lookahead_type == "General Currency Conversion":
 
-        # Get the correct balance, because WTF paypal.
-        pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
+        if typ == "Express Checkout Payment":
+            # Received money in foreign currency
+            usd = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
+            foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
+        else:
+            # Received money in foreign currency
+            usd = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
+            foreign = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
 
         exchange_rate = usd / foreign
         usd_fee = Decimal(fee) * exchange_rate
         usd_fee = Decimal(int(usd_fee * 100)) / 100
         gross = usd - usd_fee
+
+        print("      foreign: ", foreign)
+        print("  foreign fee: ", fee)
         print("          net: ", usd)
         print("      usd fee: ", usd_fee)
         print("        gross: ", gross)
         print("exchange rate: ", exchange_rate)
-        print("          net: ", net)
 
         if typ == "Express Checkout Payment":
             gross = -gross
 
         fee = usd_fee
-        net = foreign
+        net = usd
+        # Get the correct balance, because WTF paypal.
+        pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
 
+        # The balance for express checkout are zero. thanks paypal.
+        if pp_balance == Decimal(0.0):
+            print("Balance fix!")
+            pp_balance = balance - usd
 
         index += 2
     else:
-        print("*** skip non balance affecting transaction: %s", desc)
-        index += 1
-        continue
+        print(gross, fee)
+        if gross + fee == Decimal(0.0):
+            print("*** skip non balance affecting transaction: %s", desc)
+            index += 1
+            continue
+
+        print("Unknown transaction")
+        print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
+                                               str(pp_balance), str(balance)))
+        assert False
 
     if balance is None:
         balance = pp_balance - net
@@ -114,6 +136,9 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
     balance = balance + net
     if balance != pp_balance:
         print("     discrepancy: ", (pp_balance - balance))
+        sys.exit(-1)
+
+    print(desc)
 
     desc = desc.replace(",", " ")
     out.writerow([dat, desc, fee, gross])

From 8d5bcde3856ecc2ce17639b4d94419e61e69f8bd Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Tue, 24 Sep 2024 23:05:46 +0530
Subject: [PATCH 06/11] Parses everything!

---
 misc/parse-paypal-us.py | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index acf355e5..dd5f2bfa 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -35,19 +35,11 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
 lines = []
 reader = unicode_csv_reader(fp)
 for line in reader:
-
-    # Filter out lines that complicate everything
-    if line[4] in ("General Card Deposit", "Account Hold for Open Authorization", "Reversal of General Account Hold"):
-        continue
-
-    if line[5] != 'Completed':
-        continue
-
     lines.append(line)
 
 print("DATE       NAME                                           GROSS       FEE     PP_BAL    BALANCE")
 
-index = 0
+index = 1
 register = []
 balance = None
 while True:
@@ -96,6 +88,12 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
         usd_fee = Decimal(fee) * exchange_rate
         usd_fee = Decimal(int(usd_fee * 100)) / 100
         gross = usd - usd_fee
+        fee = usd_fee
+        net = usd
+
+        if typ == "Express Checkout Payment":
+            gross = -gross
+            net = -net
 
         print("      foreign: ", foreign)
         print("  foreign fee: ", fee)
@@ -104,18 +102,17 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
         print("        gross: ", gross)
         print("exchange rate: ", exchange_rate)
 
-        if typ == "Express Checkout Payment":
-            gross = -gross
 
-        fee = usd_fee
-        net = usd
         # Get the correct balance, because WTF paypal.
         pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
 
         # The balance for express checkout are zero. thanks paypal.
         if pp_balance == Decimal(0.0):
             print("Balance fix!")
-            pp_balance = balance - usd
+            print("balance: %s gross: %s" % (str(balance), str(gross)))
+            pp_balance = balance + gross
+
+        print("Balance: ", balance)
 
         index += 2
     else:
@@ -136,10 +133,10 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
     balance = balance + net
     if balance != pp_balance:
         print("     discrepancy: ", (pp_balance - balance))
+        print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
+                                               str(pp_balance), str(balance)))
         sys.exit(-1)
 
-    print(desc)
-
     desc = desc.replace(",", " ")
     out.writerow([dat, desc, fee, gross])
 

From 57a20e5fad4893c42dbb067288df5d1cc6b6b851 Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Tue, 24 Sep 2024 23:16:15 +0530
Subject: [PATCH 07/11] Finished cleaning up paypal parse script. Now not an
 utter turd and will catch import mistakes and stop.

---
 misc/parse-paypal-us.py | 252 ++++++++++++++++++++--------------------
 1 file changed, 125 insertions(+), 127 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index dd5f2bfa..0eeaf25c 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -10,149 +10,147 @@ def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
     for row in csv_reader:
         yield [cell for cell in row]
 
+def process_file(in_file, out_file, verbose):
+    out = csv.writer(_out, quoting=csv.QUOTE_MINIMAL)
+    out.writerow(["Date","Description","Amount"])
+
+    lines = []
+    reader = unicode_csv_reader(fp)
+    for line in reader:
+        lines.append(line)
+
+    print("DATE       NAME                                           GROSS       FEE     PP_BAL    BALANCE")
+
+    index = 1
+    register = []
+    balance = None
+    while True:
+        if index >= len(lines):
+            break
+
+        fields = lines[index]
+
+        desc = fields[3]
+        dat = fields[0]
+        gross = Decimal(fields[7].replace(",", ""))
+        fee = Decimal(fields[8].replace(",", ""))
+        net = Decimal(fields[9].replace(",", ""))
+        pp_balance = Decimal(fields[29].replace(",", ""))
+        status = fields[5]
+        typ = fields[4]
+        currency = fields[6]
+
+        try:
+            lookahead_type = lines[index + 1][4]
+        except IndexError:
+            lookahead_type = ""
+
+        if currency == 'USD' and typ != "General Currency Conversion":
+            # Normal native currency transactions
+            amount = gross
+
+        elif currency != 'USD' and lookahead_type == "General Currency Conversion":
+
+            if typ == "Express Checkout Payment":
+                # Received money in foreign currency
+                usd = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
+                foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
+            else:
+                # Received money in foreign currency
+                usd = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
+                foreign = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
+
+            exchange_rate = usd / foreign
+            usd_fee = Decimal(fee) * exchange_rate
+            usd_fee = Decimal(int(usd_fee * 100)) / 100
+            gross = usd - usd_fee
+            fee = usd_fee
+            net = usd
+
+            if typ == "Express Checkout Payment":
+                gross = -gross
+                net = -net
+
+            if (verbose):
+                print("      foreign: ", foreign)
+                print("  foreign fee: ", fee)
+                print("          net: ", usd)
+                print("      usd fee: ", usd_fee)
+                print("        gross: ", gross)
+                print("exchange rate: ", exchange_rate)
+
+            # Get the correct balance, because WTF paypal.
+            pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
+
+            # The balance for express checkout are zero. thanks paypal.
+            if pp_balance == Decimal(0.0):
+                if (verbose):
+                    print("Balance fix!")
+                    print("balance: %s gross: %s" % (str(balance), str(gross)))
+                pp_balance = balance + gross
+
+            index += 2
+        else:
+            if gross + fee == Decimal(0.0):
+                print("*** skip non balance affecting transaction: %s" % desc)
+                index += 1
+                continue
+
+            print("Unknown transaction")
+            print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
+                                                   str(pp_balance), str(balance)))
+            assert False
+
+        if balance is None:
+            balance = pp_balance - net
+            starting_balance = balance
+
+        balance = balance + net
+        if balance != pp_balance:
+            print("     discrepancy: ", (pp_balance - balance))
+            print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
+                                                   str(pp_balance), str(balance)))
+            assert False
+
+        desc = desc.replace(",", " ")
+        out.writerow([dat, desc, fee, gross])
+
+        print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
+                                               str(pp_balance), str(balance)))
+
+        index += 1
+
+    print("    Start paypal balance: ", starting_balance)
+    print("    Final paypal balance: ", pp_balance)
+    print("Final calculated balance: ", balance)
+
+
 if len(sys.argv) not in [3,4]:
     print("Usage parse-paypal-es.py <paypal csv file> <qbo csv file>")
     sys.exit(-1)
 
+if sys.argv[1] == "-v":
+    verbose = True
+    index = 2
+else:
+    verbose = False
+    index = 1
+
 fp = None
 try:
-    fp = open(sys.argv[1], "r")
+    fp = open(sys.argv[index], "r")
 except IOError:
     print("Cannot open input file %s" % sys.argv[1])
     sys.exit(0)
 
 _out = None
 try:
-    _out = open(sys.argv[2], "w")
+    _out = open(sys.argv[index + 1], "w")
 except IOError:
     print("Cannot open output file %s" % sys.argv[2])
     sys.exit(0)
 
-
-out = csv.writer(_out, quoting=csv.QUOTE_MINIMAL)
-out.writerow(["Date","Description","Amount"])
-
-lines = []
-reader = unicode_csv_reader(fp)
-for line in reader:
-    lines.append(line)
-
-print("DATE       NAME                                           GROSS       FEE     PP_BAL    BALANCE")
-
-index = 1
-register = []
-balance = None
-while True:
-    if index >= len(lines):
-        break
-
-    fields = lines[index]
-
-    desc = fields[3]
-    dat = fields[0]
-    gross = Decimal(fields[7].replace(",", ""))
-    fee = Decimal(fields[8].replace(",", ""))
-    net = Decimal(fields[9].replace(",", ""))
-    pp_balance = Decimal(fields[29].replace(",", ""))
-    status = fields[5]
-    typ = fields[4]
-
-#    if gross == Decimal(0.0):
-#        print("*** skip non balance affecting transaction: %s", desc)
-#        index += 1
-#        continue
-
-    currency = fields[6]
-
-    try:
-        lookahead_type = lines[index + 1][4]
-    except IndexError:
-        lookahead_type = ""
-
-    if currency == 'USD' and typ != "General Currency Conversion":
-        # Normal native currency transactions
-        amount = gross
-
-    elif currency != 'USD' and lookahead_type == "General Currency Conversion":
-
-        if typ == "Express Checkout Payment":
-            # Received money in foreign currency
-            usd = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
-            foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
-        else:
-            # Received money in foreign currency
-            usd = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
-            foreign = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
-
-        exchange_rate = usd / foreign
-        usd_fee = Decimal(fee) * exchange_rate
-        usd_fee = Decimal(int(usd_fee * 100)) / 100
-        gross = usd - usd_fee
-        fee = usd_fee
-        net = usd
-
-        if typ == "Express Checkout Payment":
-            gross = -gross
-            net = -net
-
-        print("      foreign: ", foreign)
-        print("  foreign fee: ", fee)
-        print("          net: ", usd)
-        print("      usd fee: ", usd_fee)
-        print("        gross: ", gross)
-        print("exchange rate: ", exchange_rate)
-
-
-        # Get the correct balance, because WTF paypal.
-        pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
-
-        # The balance for express checkout are zero. thanks paypal.
-        if pp_balance == Decimal(0.0):
-            print("Balance fix!")
-            print("balance: %s gross: %s" % (str(balance), str(gross)))
-            pp_balance = balance + gross
-
-        print("Balance: ", balance)
-
-        index += 2
-    else:
-        print(gross, fee)
-        if gross + fee == Decimal(0.0):
-            print("*** skip non balance affecting transaction: %s", desc)
-            index += 1
-            continue
-
-        print("Unknown transaction")
-        print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
-                                               str(pp_balance), str(balance)))
-        assert False
-
-    if balance is None:
-        balance = pp_balance - net
-
-    balance = balance + net
-    if balance != pp_balance:
-        print("     discrepancy: ", (pp_balance - balance))
-        print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
-                                               str(pp_balance), str(balance)))
-        sys.exit(-1)
-
-    desc = desc.replace(",", " ")
-    out.writerow([dat, desc, fee, gross])
-
-    print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
-                                           str(pp_balance), str(balance)))
-
-    desc = "PayPal Fee"
-    dat = fields[0]
-    if fee and Decimal(fee) != 0.0:
-        out.writerow([dat, desc, fee])
-
-    index += 1
-
-print("    Final paypal balance: ", pp_balance)
-print("Final calculated balance: ", balance)
-
+process_file(fp, _out, verbose)
 
 fp.close()
 _out.close()

From f7a1a4a1e9c6326e49cc9667147a12cc19ac8e48 Mon Sep 17 00:00:00 2001
From: Akira <5215000+ApeKattQuest-MonkeyPython@users.noreply.github.com>
Date: Thu, 26 Sep 2024 11:51:53 +0530
Subject: [PATCH 08/11] Updating MonkeyPython  (Akira's) Bio

---
 metabrainz/templates/index/team.html | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/metabrainz/templates/index/team.html b/metabrainz/templates/index/team.html
index ebb0201e..e1740636 100644
--- a/metabrainz/templates/index/team.html
+++ b/metabrainz/templates/index/team.html
@@ -294,23 +294,23 @@ <h4>{{ _('<span class="name">David "drsaunde" Saunders</span> &mdash; MusicBrain
 
     <div class="person">
       <img
-        alt="CatCat's photo"
+        alt="MonkeyPython's photo"
         class="pic"
         src="{{ url_for('static', filename='img/team/catcat.jpg') }}"
       />
       <div class="bio">
         <h4>{{ _('<span class="name">Akira Holm</span> &mdash; MusicBrainz Instruments') }}</h4>
         <p>
-          {{ _('CatQuest, the Endeavouring Cat! (CatCat, for short) has been a MusicBrainz editor since 2004, and
+          {{ _('ApeKattQuest or MonkeyPython has been a MusicBrainz editor since 2004, and
           is in charge of adding and improving the musical instruments in MusicBrainz (the very unseriously-named,
           yet quite important "Instrument Inserter" position). When not looking all over the internet (and the library!)
           for instrument data, his other hobbies include playing games, listening to all sorts of different music, Lupin 
           III, colouring, and collecting all sorts of things, from stickers and stamps to bottlecaps.') }}
         </p>
         <ul class="links">
-          <li><strong>IRC:</strong> ApeKattQuest</li>
-          <li><strong>MusicBrainz:</strong> <a href="https://musicbrainz.org/user/CatCat">CatCat</a></li>
-          <li><strong>ListenBrainz:</strong> <a href="https://listenbrainz.org/user/CatCat">CatCat</a></li>
+          <li><strong>IRC:</strong> ApeKattQuest, MonkeyPython (plus a whole lot of other Lupin III references)</li>
+          <li><strong>MusicBrainz:</strong> <a href="https://musicbrainz.org/user/ApeKattQuest, MonkeyPython">ApeKattQuest, MonkeyPython</a></li>
+          <li><strong>ListenBrainz:</strong> <a href="https://listenbrainz.org/user/ApeKattQuest, MonkeyPython">ApeKattQuest, MonkeyPython</a></li>
         </ul>
       </div>
     </div>

From a331dbb1711d24462c47fe81f499448fe8e00909 Mon Sep 17 00:00:00 2001
From: Kartik Ohri <kartikohri13@gmail.com>
Date: Mon, 14 Oct 2024 19:59:40 -0600
Subject: [PATCH 09/11] interim check-in

---
 misc/parse-paypal-us.py | 426 +++++++++++++++++++++++++---------------
 1 file changed, 273 insertions(+), 153 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index 0eeaf25c..67f58892 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -1,156 +1,276 @@
 #!/usr/bin/env python3
-
-import sys
-import re
+import argparse
+from dataclasses import dataclass
 import csv
-from decimal import Decimal, InvalidOperation
-
-def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
-    csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
-    for row in csv_reader:
-        yield [cell for cell in row]
-
-def process_file(in_file, out_file, verbose):
-    out = csv.writer(_out, quoting=csv.QUOTE_MINIMAL)
-    out.writerow(["Date","Description","Amount"])
-
-    lines = []
-    reader = unicode_csv_reader(fp)
-    for line in reader:
-        lines.append(line)
-
-    print("DATE       NAME                                           GROSS       FEE     PP_BAL    BALANCE")
-
-    index = 1
-    register = []
-    balance = None
-    while True:
-        if index >= len(lines):
-            break
-
-        fields = lines[index]
-
-        desc = fields[3]
-        dat = fields[0]
-        gross = Decimal(fields[7].replace(",", ""))
-        fee = Decimal(fields[8].replace(",", ""))
-        net = Decimal(fields[9].replace(",", ""))
-        pp_balance = Decimal(fields[29].replace(",", ""))
-        status = fields[5]
-        typ = fields[4]
-        currency = fields[6]
-
-        try:
-            lookahead_type = lines[index + 1][4]
-        except IndexError:
-            lookahead_type = ""
-
-        if currency == 'USD' and typ != "General Currency Conversion":
-            # Normal native currency transactions
-            amount = gross
-
-        elif currency != 'USD' and lookahead_type == "General Currency Conversion":
-
-            if typ == "Express Checkout Payment":
-                # Received money in foreign currency
-                usd = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
-                foreign = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
+from datetime import datetime
+from decimal import Decimal
+from typing import Dict
+
+from ofxtools.models import STMTTRN, BANKTRANLIST, STMTRS, BANKMSGSRSV1, OFX, BANKACCTFROM, LEDGERBAL, STMTTRNRS, \
+    STATUS, FI, SONRS, SIGNONMSGSRSV1
+from ofxtools.utils import UTC
+import xml.etree.ElementTree as ET
+
+
+@dataclass
+class Transaction:
+    description: str
+    date: datetime
+    gross: Decimal
+    fee: Decimal
+    net: Decimal
+    currency: str
+    type: str
+    balance: Decimal
+    comment: str
+    is_donation: bool
+    accounted_for: bool
+    line: str
+    balance_impact: str
+    transaction_id: str
+    reference_transaction_id: str
+    subtransactions: list
+
+
+def classify_transaction(transaction: Transaction):
+    if transaction.is_donation and transaction.currency == 'USD':
+        return {
+            'trntype': 'CREDIT',
+            'trnamt': transaction.gross,
+            'name': 'Donation',
+            'memo': '4010 Income - Donations - PayPal'
+        }
+    elif transaction.is_donation and transaction.currency != 'USD':
+        return {
+            'trntype': 'CREDIT',
+            'trnamt': transaction.gross,
+            'name': None,  # Not classified
+            'memo': f'PayPal donation in {transaction.currency}. Reference: {transaction.reference_transaction_id}'
+        }
+    elif transaction.type == 'Payment':
+        return {
+            'trntype': 'DEBIT',
+            'trnamt': -transaction.gross,
+            'name': None,  # Not classified
+            'memo': f'Payment in {transaction.currency}'
+        }
+    elif transaction.type == 'Transfer':
+        return {
+            'trntype': 'XFER',
+            'trnamt': -transaction.gross,
+            'name': '3100 Bank - PPB Checking',
+            'memo': 'Transfer to Pacific Premier'
+        }
+    else:  # Checkout payments, refunds, etc.
+        return {
+            'trntype': 'OTHER',
+            'trnamt': transaction.gross,
+            'name': None,  # Not classified
+            'memo': f'Unclassified transaction: {transaction.type}'
+        }
+
+
+def create_ofx(transactions: Dict[str, Transaction]):
+    statement_transactions = []
+    for transaction in list(transactions.values()):
+        classified = classify_transaction(transaction)
+
+        if not classified:
+            continue
+        tran = STMTTRN(
+            trntype=classified['trntype'],
+            dtposted=transaction.date.replace(tzinfo=UTC),
+            trnamt=classified['trnamt'],
+            fitid=transaction.transaction_id,
+            name=classified['name'],
+            memo=classified['memo']
+        )
+        statement_transactions.append(tran)
+
+        if transaction.fee != 0:
+            fee_tran = STMTTRN(
+                trntype='DEBIT',
+                dtposted=transaction.date.replace(tzinfo=UTC),
+                trnamt=transaction.fee,
+                fitid=f"{transaction.transaction_id}_fee",
+                name=f'PayPal fee for {transaction.transaction_id}',
+                memo='5020 Expense - Bank - PayPal'
+            )
+            statement_transactions.append(fee_tran)
+
+        for subtrans in transaction.subtransactions:
+            sub_tran = STMTTRN(
+                trntype='OTHER',
+                dtposted=transaction.date.replace(tzinfo=UTC),
+                trnamt=subtrans.gross,
+                fitid=f"{transaction.transaction_id}",
+                name=None,
+                memo=f"Subtransaction: {subtrans.description}"
+            )
+            statement_transactions.append(sub_tran)
+
+    tl = BANKTRANLIST(*statement_transactions,
+                      dtstart=statement_transactions[0].dtposted,
+                      dtend=statement_transactions[-1].dtposted)
+
+    t = list(transactions.values())[0]
+    stmtrs = STMTRS(
+        curdef='USD',
+        bankacctfrom=BANKACCTFROM(bankid='Paypal', acctid='3020', accttype='CHECKING'),
+        banktranlist=tl,
+        ledgerbal=LEDGERBAL(balamt=t.balance, dtasof=t.date.replace(tzinfo=UTC)),
+    )
+    msgsrs = BANKMSGSRSV1(STMTTRNRS(
+        stmtrs=stmtrs,
+        trnuid="0",
+        status=STATUS(code="0", severity="INFO"))
+    )
+    fi = FI(org='Illuminati', fid='666')
+    sonrs = SONRS(status=STATUS(code="0", severity="INFO"), dtserver = datetime(2015, 1, 2, 17, tzinfo=UTC),
+                  language = 'ENG', fi = fi)
+    signonmsgs = SIGNONMSGSRSV1(sonrs=sonrs)
+    return OFX(signonmsgsrsv1=signonmsgs, bankmsgsrsv1=msgsrs)
+
+
+def parse(input_file) -> Dict[str, Transaction]:
+
+    transactions = {}
+
+    with open(input_file, "r", encoding="utf-8-sig") as f:
+        csv_reader = csv.DictReader(f, dialect="excel")
+        for line in csv_reader:
+            if line["Option 1 Name"] == "is_donation":
+                is_donation = line["Option 1 Value"] == "yes"
+            elif line["Option 2 Name"] == "is_donation":
+                is_donation = line["Option 2 Value"] == "yes"
+            elif "donation" in line["Subject"].lower():
+                is_donation = True
             else:
-                # Received money in foreign currency
-                usd = Decimal(lines[index + 2][7].replace(",", "")).copy_abs()
-                foreign = Decimal(lines[index + 1][7].replace(",", "")).copy_abs()
-
-            exchange_rate = usd / foreign
-            usd_fee = Decimal(fee) * exchange_rate
-            usd_fee = Decimal(int(usd_fee * 100)) / 100
-            gross = usd - usd_fee
-            fee = usd_fee
-            net = usd
-
-            if typ == "Express Checkout Payment":
-                gross = -gross
-                net = -net
-
-            if (verbose):
-                print("      foreign: ", foreign)
-                print("  foreign fee: ", fee)
-                print("          net: ", usd)
-                print("      usd fee: ", usd_fee)
-                print("        gross: ", gross)
-                print("exchange rate: ", exchange_rate)
-
-            # Get the correct balance, because WTF paypal.
-            pp_balance = Decimal(lines[index + 2][29].replace(",", ""))
-
-            # The balance for express checkout are zero. thanks paypal.
-            if pp_balance == Decimal(0.0):
-                if (verbose):
-                    print("Balance fix!")
-                    print("balance: %s gross: %s" % (str(balance), str(gross)))
-                pp_balance = balance + gross
-
-            index += 2
-        else:
-            if gross + fee == Decimal(0.0):
-                print("*** skip non balance affecting transaction: %s" % desc)
-                index += 1
-                continue
-
-            print("Unknown transaction")
-            print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
-                                                   str(pp_balance), str(balance)))
-            assert False
-
-        if balance is None:
-            balance = pp_balance - net
-            starting_balance = balance
-
-        balance = balance + net
-        if balance != pp_balance:
-            print("     discrepancy: ", (pp_balance - balance))
-            print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
-                                                   str(pp_balance), str(balance)))
-            assert False
-
-        desc = desc.replace(",", " ")
-        out.writerow([dat, desc, fee, gross])
-
-        print("%s %-40s %10s %10s %10s %10s" % (dat, desc, str(gross), str(fee),
-                                               str(pp_balance), str(balance)))
-
-        index += 1
-
-    print("    Start paypal balance: ", starting_balance)
-    print("    Final paypal balance: ", pp_balance)
-    print("Final calculated balance: ", balance)
-
-
-if len(sys.argv) not in [3,4]:
-    print("Usage parse-paypal-es.py <paypal csv file> <qbo csv file>")
-    sys.exit(-1)
-
-if sys.argv[1] == "-v":
-    verbose = True
-    index = 2
-else:
-    verbose = False
-    index = 1
-
-fp = None
-try:
-    fp = open(sys.argv[index], "r")
-except IOError:
-    print("Cannot open input file %s" % sys.argv[1])
-    sys.exit(0)
-
-_out = None
-try:
-    _out = open(sys.argv[index + 1], "w")
-except IOError:
-    print("Cannot open output file %s" % sys.argv[2])
-    sys.exit(0)
-
-process_file(fp, _out, verbose)
-
-fp.close()
-_out.close()
+                is_donation = False
+
+            transaction = Transaction(
+                description=line["Name"],
+                date=datetime.strptime(line["Date"], "%m/%d/%Y"),
+                gross=Decimal(line["Gross"].replace(",", "")),
+                fee=Decimal(line["Fee"].replace(",", "")),
+                net=Decimal(line["Net"].replace(",", "")),
+                currency=line["Currency"],
+                type=line["Type"],
+                balance=Decimal(line["Balance"].replace(",", "")),
+                is_donation=bool(is_donation),  # is_donation
+                comment="",
+                accounted_for=False,
+                balance_impact=line["Balance Impact"],
+                transaction_id=line["Transaction ID"],
+                reference_transaction_id=line["Reference Txn ID"],
+                subtransactions=[],
+                line=line
+            )
+            transactions[transaction.transaction_id] = transaction
+
+    subtransaction_keys = []
+    for transaction in transactions.values():
+        if transaction.reference_transaction_id is not None and transaction.reference_transaction_id != "" \
+                and transaction.reference_transaction_id in transactions:
+            main_transaction = transactions[transaction.reference_transaction_id]
+            main_transaction.subtransactions.append(transaction)
+            subtransaction_keys.append(transaction.transaction_id)
+
+    for subtransaction_key in subtransaction_keys:
+        subtransaction = transactions[subtransaction_key]
+
+        if subtransaction.subtransactions:
+            print(subtransaction)
+            for t in subtransaction.subtransactions:
+                print("\t", t)
+            transactions.pop(subtransaction_key)
+
+    return transactions
+
+    # usd_donations = []
+    # usd_payments = []
+    # withdrawals = []
+    # usd_expenses = []
+    #
+    # non_usd_donations = {}
+    # non_usd_expenses = {}
+    # non_usd_payments = {}
+    #
+    # index = 0
+    # while index < len(transactions):
+    #     transaction = transactions[index]
+    #     if transaction.type == "General Currency Conversion":
+    #         if non_usd_expenses.get(transaction.reference_transaction_id):
+    #             main_transaction = non_usd_expenses[transaction.reference_transaction_id]
+    #         elif non_usd_payments.get(transaction.reference_transaction_id):
+    #             main_transaction = non_usd_payments[transaction.reference_transaction_id]
+    #         elif non_usd_donations.get(transaction.reference_transaction_id):
+    #             main_transaction = non_usd_donations[transaction.reference_transaction_id]
+    #         else:
+    #             main_transaction = None
+    #
+    #         if main_transaction is not None:
+    #             main_transaction.subtransactions.append(transaction)
+    #             transaction.accounted_for = True
+    #         else:
+    #             print(transaction.line)
+    #
+    #         index += 1
+    #     elif transaction.currency == "USD":
+    #         if transaction.is_donation:
+    #             usd_donations.append(transaction)
+    #             transaction.accounted_for = True
+    #         elif transaction.type == "General Withdrawal":
+    #             withdrawals.append(transaction)
+    #             transaction.accounted_for = True
+    #         elif transaction.type in ["General Payment", "Subscription Payment", "Website Payment", "Reversal of General Account Hold"] and transaction.balance_impact == "Credit":
+    #             usd_payments.append(transaction)
+    #             transaction.accounted_for = True
+    #         elif transaction.balance_impact == "Debit":
+    #             usd_expenses.append(transaction)
+    #             transaction.accounted_for = True
+    #         else:
+    #             print(transaction.line)
+    #         index += 1
+    #     else:
+    #         if transaction.is_donation:
+    #             non_usd_donations[transaction.transaction_id] = transaction
+    #             transaction.accounted_for = True
+    #         elif transaction.type in ["Express Checkout Payment", "General Payment"] and transaction.balance_impact == "Credit":
+    #             non_usd_payments[transaction.transaction_id] = transaction
+    #             transaction.accounted_for = True
+    #         elif transaction.balance_impact == "Debit":
+    #             non_usd_expenses[transaction.transaction_id] = transaction
+    #             transaction.accounted_for = True
+    #         else:
+    #             print(transaction.line)
+    #         index += 1
+    #
+    # print("all transactions", len(transactions))
+    #
+    # count = 0
+    # for transaction in transactions:
+    #     if transaction.accounted_for:
+    #         count += 1
+    #
+    # print("accounted for", count)
+
+
+def main():
+    parser = argparse.ArgumentParser(prog='Parse PayPal Transactions')
+    parser.add_argument("--input", help="Paypal transactions/activity csv file", required=True)
+    parser.add_argument("--output", help="Desired simplified report file path", required=True)
+
+    args = parser.parse_args()
+    transactions = parse(args.input)
+
+    ofx = create_ofx(transactions)
+
+    root = ofx.to_etree()
+    message = ET.tostring(root).decode()
+    with open(args.output, 'w') as ofxfile:
+        ofxfile.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?><?OFX OFXHEADER="200" VERSION="220" SECURITY="NONE" OLDFILEUID="NONE" NEWFILEUID="NONE"?>')
+        ofxfile.write(message)
+
+if __name__ == "__main__":
+    main()

From f3bee837de97d39ee93a20c3c4b45fc26f83aa9c Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Thu, 17 Oct 2024 19:22:12 +0200
Subject: [PATCH 10/11] Back to CSV, but making progress.

---
 misc/parse-paypal-us.py | 290 ++++++++++++++--------------------------
 1 file changed, 97 insertions(+), 193 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index 67f58892..915c958f 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -3,15 +3,15 @@
 from dataclasses import dataclass
 import csv
 from datetime import datetime
-from decimal import Decimal
+from decimal import Decimal, ROUND_DOWN
 from typing import Dict
+from icecream import ic
 
 from ofxtools.models import STMTTRN, BANKTRANLIST, STMTRS, BANKMSGSRSV1, OFX, BANKACCTFROM, LEDGERBAL, STMTTRNRS, \
     STATUS, FI, SONRS, SIGNONMSGSRSV1
 from ofxtools.utils import UTC
 import xml.etree.ElementTree as ET
 
-
 @dataclass
 class Transaction:
     description: str
@@ -28,113 +28,86 @@ class Transaction:
     line: str
     balance_impact: str
     transaction_id: str
+    invoice_num: str
     reference_transaction_id: str
     subtransactions: list
 
 
-def classify_transaction(transaction: Transaction):
-    if transaction.is_donation and transaction.currency == 'USD':
-        return {
-            'trntype': 'CREDIT',
-            'trnamt': transaction.gross,
-            'name': 'Donation',
-            'memo': '4010 Income - Donations - PayPal'
-        }
-    elif transaction.is_donation and transaction.currency != 'USD':
-        return {
-            'trntype': 'CREDIT',
-            'trnamt': transaction.gross,
-            'name': None,  # Not classified
-            'memo': f'PayPal donation in {transaction.currency}. Reference: {transaction.reference_transaction_id}'
-        }
-    elif transaction.type == 'Payment':
-        return {
-            'trntype': 'DEBIT',
-            'trnamt': -transaction.gross,
-            'name': None,  # Not classified
-            'memo': f'Payment in {transaction.currency}'
-        }
-    elif transaction.type == 'Transfer':
-        return {
-            'trntype': 'XFER',
-            'trnamt': -transaction.gross,
-            'name': '3100 Bank - PPB Checking',
-            'memo': 'Transfer to Pacific Premier'
-        }
-    else:  # Checkout payments, refunds, etc.
-        return {
-            'trntype': 'OTHER',
-            'trnamt': transaction.gross,
-            'name': None,  # Not classified
-            'memo': f'Unclassified transaction: {transaction.type}'
-        }
-
-
-def create_ofx(transactions: Dict[str, Transaction]):
-    statement_transactions = []
-    for transaction in list(transactions.values()):
-        classified = classify_transaction(transaction)
-
-        if not classified:
-            continue
-        tran = STMTTRN(
-            trntype=classified['trntype'],
-            dtposted=transaction.date.replace(tzinfo=UTC),
-            trnamt=classified['trnamt'],
-            fitid=transaction.transaction_id,
-            name=classified['name'],
-            memo=classified['memo']
-        )
-        statement_transactions.append(tran)
-
-        if transaction.fee != 0:
-            fee_tran = STMTTRN(
-                trntype='DEBIT',
-                dtposted=transaction.date.replace(tzinfo=UTC),
-                trnamt=transaction.fee,
-                fitid=f"{transaction.transaction_id}_fee",
-                name=f'PayPal fee for {transaction.transaction_id}',
-                memo='5020 Expense - Bank - PayPal'
-            )
-            statement_transactions.append(fee_tran)
-
-        for subtrans in transaction.subtransactions:
-            sub_tran = STMTTRN(
-                trntype='OTHER',
-                dtposted=transaction.date.replace(tzinfo=UTC),
-                trnamt=subtrans.gross,
-                fitid=f"{transaction.transaction_id}",
-                name=None,
-                memo=f"Subtransaction: {subtrans.description}"
-            )
-            statement_transactions.append(sub_tran)
-
-    tl = BANKTRANLIST(*statement_transactions,
-                      dtstart=statement_transactions[0].dtposted,
-                      dtend=statement_transactions[-1].dtposted)
-
-    t = list(transactions.values())[0]
-    stmtrs = STMTRS(
-        curdef='USD',
-        bankacctfrom=BANKACCTFROM(bankid='Paypal', acctid='3020', accttype='CHECKING'),
-        banktranlist=tl,
-        ledgerbal=LEDGERBAL(balamt=t.balance, dtasof=t.date.replace(tzinfo=UTC)),
-    )
-    msgsrs = BANKMSGSRSV1(STMTTRNRS(
-        stmtrs=stmtrs,
-        trnuid="0",
-        status=STATUS(code="0", severity="INFO"))
-    )
-    fi = FI(org='Illuminati', fid='666')
-    sonrs = SONRS(status=STATUS(code="0", severity="INFO"), dtserver = datetime(2015, 1, 2, 17, tzinfo=UTC),
-                  language = 'ENG', fi = fi)
-    signonmsgs = SIGNONMSGSRSV1(sonrs=sonrs)
-    return OFX(signonmsgsrsv1=signonmsgs, bankmsgsrsv1=msgsrs)
+def create_csv_file(out_file: str, transactions: Dict[str, Transaction]):
+
+    balance = None
+    try:
+        _out = open(out_file, "w")
+    except IOError:
+        print("Cannot open output file %s" % sys.argv[2])
+        sys.exit(0)
+
+    out = csv.writer(_out, quoting=csv.QUOTE_MINIMAL)
+    out.writerow(["Date","Description","Amount"])
+    for t in transactions:
+
+        if t.type in ("General Withdrawal", "User Initiated Withdrawal"):
+            t.description = "Transfer"
+        elif t.type.startswith("Bank Deposit to PP Account"):
+            t.description = "Bank Deposit to PP Account"
+        elif t.type == "General Card Deposit":
+            t.description = "General Card Deposit"
+
+        if t.currency != "USD":
+            pp_balance = None
+            for sub_t in t.subtransactions:
+                if sub_t.currency == "USD" and sub_t.type == "General Currency Conversion":
+                    native = sub_t.gross
+                    pp_balance = sub_t.balance
+                if sub_t.currency != "USD":
+                    foreign = -sub_t.gross
+
+            exchange_rate = native / foreign
+            usd_fee = Decimal(t.fee * exchange_rate).quantize(Decimal('.01'), rounding=ROUND_DOWN)
+
+            print(f" native: {native:.2f}")
+            print(f"foreign: {foreign:.2f}")
+            print(f"eur fee: {t.fee:.2f}")
+            print(f"usd fee: {usd_fee:.2f}")
+            print(f" x-rate: {exchange_rate:.10f}")
+
+            t.gross = native - usd_fee
+            t.fee = usd_fee
+            ic(t)
+            print(f" fix bal: {pp_balance:.2f}")
+            t.balance = pp_balance 
+            print()
+        else:
+            print(f"  gross: {t.gross:.2f}")
+            print(f"    fee: {t.fee:.2f}")
+            print(f"balance: {t.balance:.2f}")
+            print()
+
+        if balance is None:
+            balance = t.balance
+        else:
+            balance += t.gross + t.fee
+            if t.balance != balance:
+                print(f"    balance: {balance:.2f}")
+                print(f" pp balance: {t.balance:.2f}")
+                print(f"Discrepancy: {t.balance - balance:.2f}")
+                ic(t)
+                assert False
+
+        assert t.gross != Decimal(0.0)
+        if t.description == "":
+            print(t)
+            assert False
+        out.writerow([t.date.strftime("%m/%d/%Y"), t.description, t.gross])
+        if t.fee != Decimal(0.0):
+            out.writerow([t.date.strftime("%m/%d/%Y"), "PayPal Fee", t.fee])
+
+    _out.close()
 
 
 def parse(input_file) -> Dict[str, Transaction]:
 
-    transactions = {}
+    transactions = []
 
     with open(input_file, "r", encoding="utf-8-sig") as f:
         csv_reader = csv.DictReader(f, dialect="excel")
@@ -150,7 +123,7 @@ def parse(input_file) -> Dict[str, Transaction]:
 
             transaction = Transaction(
                 description=line["Name"],
-                date=datetime.strptime(line["Date"], "%m/%d/%Y"),
+                date=datetime.strptime(line["Date"] + " " + line["Time"], "%m/%d/%Y %H:%M:%S"),
                 gross=Decimal(line["Gross"].replace(",", "")),
                 fee=Decimal(line["Fee"].replace(",", "")),
                 net=Decimal(line["Net"].replace(",", "")),
@@ -162,98 +135,36 @@ def parse(input_file) -> Dict[str, Transaction]:
                 accounted_for=False,
                 balance_impact=line["Balance Impact"],
                 transaction_id=line["Transaction ID"],
+                invoice_num=line["Invoice Number"],
                 reference_transaction_id=line["Reference Txn ID"],
                 subtransactions=[],
                 line=line
             )
-            transactions[transaction.transaction_id] = transaction
+            transactions.append(transaction)
 
-    subtransaction_keys = []
-    for transaction in transactions.values():
-        if transaction.reference_transaction_id is not None and transaction.reference_transaction_id != "" \
-                and transaction.reference_transaction_id in transactions:
-            main_transaction = transactions[transaction.reference_transaction_id]
-            main_transaction.subtransactions.append(transaction)
-            subtransaction_keys.append(transaction.transaction_id)
+    subbed = []
+    for i, t in enumerate(transactions):
+        if i == 0:
+            subbed.append(t)
+            continue
 
-    for subtransaction_key in subtransaction_keys:
-        subtransaction = transactions[subtransaction_key]
+        prev = subbed[-1]
 
-        if subtransaction.subtransactions:
-            print(subtransaction)
-            for t in subtransaction.subtransactions:
-                print("\t", t)
-            transactions.pop(subtransaction_key)
+        # If they snuck a card deposit into a transaction, process it first as a separate transaction!
+        if prev.date == t.date and t.description == "" and t.type == "General Card Deposit":
+            subbed.insert(len(subbed) - 1, t)
+            continue
 
-    return transactions
+        if prev.date == t.date and t.description == "":
+            prev.subtransactions.append(t)
+            continue
 
-    # usd_donations = []
-    # usd_payments = []
-    # withdrawals = []
-    # usd_expenses = []
-    #
-    # non_usd_donations = {}
-    # non_usd_expenses = {}
-    # non_usd_payments = {}
-    #
-    # index = 0
-    # while index < len(transactions):
-    #     transaction = transactions[index]
-    #     if transaction.type == "General Currency Conversion":
-    #         if non_usd_expenses.get(transaction.reference_transaction_id):
-    #             main_transaction = non_usd_expenses[transaction.reference_transaction_id]
-    #         elif non_usd_payments.get(transaction.reference_transaction_id):
-    #             main_transaction = non_usd_payments[transaction.reference_transaction_id]
-    #         elif non_usd_donations.get(transaction.reference_transaction_id):
-    #             main_transaction = non_usd_donations[transaction.reference_transaction_id]
-    #         else:
-    #             main_transaction = None
-    #
-    #         if main_transaction is not None:
-    #             main_transaction.subtransactions.append(transaction)
-    #             transaction.accounted_for = True
-    #         else:
-    #             print(transaction.line)
-    #
-    #         index += 1
-    #     elif transaction.currency == "USD":
-    #         if transaction.is_donation:
-    #             usd_donations.append(transaction)
-    #             transaction.accounted_for = True
-    #         elif transaction.type == "General Withdrawal":
-    #             withdrawals.append(transaction)
-    #             transaction.accounted_for = True
-    #         elif transaction.type in ["General Payment", "Subscription Payment", "Website Payment", "Reversal of General Account Hold"] and transaction.balance_impact == "Credit":
-    #             usd_payments.append(transaction)
-    #             transaction.accounted_for = True
-    #         elif transaction.balance_impact == "Debit":
-    #             usd_expenses.append(transaction)
-    #             transaction.accounted_for = True
-    #         else:
-    #             print(transaction.line)
-    #         index += 1
-    #     else:
-    #         if transaction.is_donation:
-    #             non_usd_donations[transaction.transaction_id] = transaction
-    #             transaction.accounted_for = True
-    #         elif transaction.type in ["Express Checkout Payment", "General Payment"] and transaction.balance_impact == "Credit":
-    #             non_usd_payments[transaction.transaction_id] = transaction
-    #             transaction.accounted_for = True
-    #         elif transaction.balance_impact == "Debit":
-    #             non_usd_expenses[transaction.transaction_id] = transaction
-    #             transaction.accounted_for = True
-    #         else:
-    #             print(transaction.line)
-    #         index += 1
-    #
-    # print("all transactions", len(transactions))
-    #
-    # count = 0
-    # for transaction in transactions:
-    #     if transaction.accounted_for:
-    #         count += 1
-    #
-    # print("accounted for", count)
+        subbed.append(t)
+
+#    for t in subbed:
+#        ic(t)
+
+    return subbed
 
 
 def main():
@@ -263,14 +174,7 @@ def main():
 
     args = parser.parse_args()
     transactions = parse(args.input)
-
-    ofx = create_ofx(transactions)
-
-    root = ofx.to_etree()
-    message = ET.tostring(root).decode()
-    with open(args.output, 'w') as ofxfile:
-        ofxfile.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?><?OFX OFXHEADER="200" VERSION="220" SECURITY="NONE" OLDFILEUID="NONE" NEWFILEUID="NONE"?>')
-        ofxfile.write(message)
+    create_csv_file(args.output, transactions)
 
 if __name__ == "__main__":
     main()

From 246da5dc019cfaceff1475c5007a964f893ac508 Mon Sep 17 00:00:00 2001
From: Robert Kaye <rob@musicbrainz.org>
Date: Fri, 18 Oct 2024 18:18:33 +0200
Subject: [PATCH 11/11] Reverted back to CSV, but now the CSV files import
 cleanly without discrepancies!

---
 misc/parse-paypal-us.py | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/misc/parse-paypal-us.py b/misc/parse-paypal-us.py
index 915c958f..0f9e0d03 100755
--- a/misc/parse-paypal-us.py
+++ b/misc/parse-paypal-us.py
@@ -65,23 +65,23 @@ def create_csv_file(out_file: str, transactions: Dict[str, Transaction]):
             exchange_rate = native / foreign
             usd_fee = Decimal(t.fee * exchange_rate).quantize(Decimal('.01'), rounding=ROUND_DOWN)
 
-            print(f" native: {native:.2f}")
-            print(f"foreign: {foreign:.2f}")
-            print(f"eur fee: {t.fee:.2f}")
-            print(f"usd fee: {usd_fee:.2f}")
-            print(f" x-rate: {exchange_rate:.10f}")
+#            print(f" native: {native:.2f}")
+#            print(f"foreign: {foreign:.2f}")
+#            print(f"eur fee: {t.fee:.2f}")
+#            print(f"usd fee: {usd_fee:.2f}")
+#            print(f" x-rate: {exchange_rate:.10f}")
 
             t.gross = native - usd_fee
             t.fee = usd_fee
-            ic(t)
-            print(f" fix bal: {pp_balance:.2f}")
+#            ic(t)
+#            print(f" fix bal: {pp_balance:.2f}")
             t.balance = pp_balance 
-            print()
-        else:
-            print(f"  gross: {t.gross:.2f}")
-            print(f"    fee: {t.fee:.2f}")
-            print(f"balance: {t.balance:.2f}")
-            print()
+#            print()
+#        else:
+#            print(f"  gross: {t.gross:.2f}")
+#            print(f"    fee: {t.fee:.2f}")
+#            print(f"balance: {t.balance:.2f}")
+#            print()
 
         if balance is None:
             balance = t.balance
@@ -94,6 +94,8 @@ def create_csv_file(out_file: str, transactions: Dict[str, Transaction]):
                 ic(t)
                 assert False
 
+        print(t.date, t.balance, balance)
+
         assert t.gross != Decimal(0.0)
         if t.description == "":
             print(t)
@@ -102,6 +104,8 @@ def create_csv_file(out_file: str, transactions: Dict[str, Transaction]):
         if t.fee != Decimal(0.0):
             out.writerow([t.date.strftime("%m/%d/%Y"), "PayPal Fee", t.fee])
 
+    print(f"Final balance: {balance:.2f}")
+
     _out.close()
 
 
@@ -150,11 +154,20 @@ def parse(input_file) -> Dict[str, Transaction]:
 
         prev = subbed[-1]
 
+        # If the balance doesn't change, ignore it
+        if t.net == Decimal(0.0):
+            continue
+
         # If they snuck a card deposit into a transaction, process it first as a separate transaction!
         if prev.date == t.date and t.description == "" and t.type == "General Card Deposit":
             subbed.insert(len(subbed) - 1, t)
             continue
 
+        if prev.currency != "USD" and prev.date == t.date and \
+            t.description == "PayPal" and t.type == "Reversal of General Account Hold":
+            subbed.insert(len(subbed) - 1, t)
+            continue
+
         if prev.date == t.date and t.description == "":
             prev.subtransactions.append(t)
             continue