From 49249a2ce523aa4936617cf79ebdc59a10f0df71 Mon Sep 17 00:00:00 2001 From: Smit Joshi Date: Wed, 12 Aug 2026 16:00:33 +0530 Subject: [PATCH 1/2] fix(dedup): scope content hashes by account --- README.md | 5 +++-- statement_normalizer/dedup.py | 11 ++++++----- statement_normalizer/schema.py | 23 ++++++++++++----------- tests/test_dedup.py | 13 +++++++++++-- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 06cb513..e9dc9dc 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,9 @@ positive) regardless of how the source expressed direction. `fitid`, `account_id`, and currency. - De-duplicates transactions with deterministic heuristics: - Bank-assigned `FITID` (from OFX) is the authoritative key when present. - - Otherwise a content hash over (date, signed amount, currency, canonical - description), while preserving legitimately-repeated same-day charges. + - Otherwise a content hash over (account ID, date, signed amount, currency, + canonical description), while preserving legitimately-repeated same-day + charges. - Merges multiple statements (e.g. overlapping monthly exports) into one de-duplicated list. - Ships a CLI and a small Python API. diff --git a/statement_normalizer/dedup.py b/statement_normalizer/dedup.py index 654fcbf..90229ec 100644 --- a/statement_normalizer/dedup.py +++ b/statement_normalizer/dedup.py @@ -10,11 +10,12 @@ * If two transactions share a non-empty ``fitid`` (and account), they are the same transaction. FITID is bank-assigned and authoritative. -* Otherwise fall back to a content hash over (date, signed amount, currency, - canonical description). To avoid collapsing genuinely-repeated charges (e.g. - two identical $4.75 coffees on the same day), identical content rows are only - treated as duplicates beyond the count seen on a *single* source statement — - i.e. we keep the max multiplicity observed within any one input, not the sum. +* Otherwise fall back to a content hash over (account, date, signed amount, + currency, canonical description). To avoid collapsing genuinely-repeated + charges (e.g. two identical $4.75 coffees on the same day), identical content + rows are only treated as duplicates beyond the count seen on a *single* source + statement — i.e. we keep the max multiplicity observed within any one input, + not the sum. The first occurrence is kept; order is otherwise preserved (stable). """ diff --git a/statement_normalizer/schema.py b/statement_normalizer/schema.py index 7808f9f..863387b 100644 --- a/statement_normalizer/schema.py +++ b/statement_normalizer/schema.py @@ -120,18 +120,19 @@ def canonical_description(self) -> str: def content_hash(self) -> str: """Stable hash over the identity-bearing fields. - Two rows with the same date, signed amount, currency and canonical - description are considered the same transaction when no FITID is - available. Used by the dedup heuristics. + Two rows from the same account with the same date, signed amount, + currency and canonical description are considered the same transaction + when no FITID is available. Used by the dedup heuristics. """ - key = "|".join( - [ - self.date.isoformat(), - f"{self.amount:.2f}", - self.currency, - self.canonical_description, - ] - ) + fields = [ + self.date.isoformat(), + f"{self.amount:.2f}", + self.currency, + self.canonical_description, + ] + if self.account_id: + fields.insert(0, self.account_id) + key = "|".join(fields) return hashlib.sha256(key.encode("utf-8")).hexdigest() def to_dict(self) -> dict[str, Any]: diff --git a/tests/test_dedup.py b/tests/test_dedup.py index d1dc286..586ae16 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -25,12 +25,21 @@ def test_fitid_dedup(): def test_content_dedup_collapses_without_multiplicity(): - a = _txn(5, "-4.75", "COFFEE ROASTERS") - b = _txn(5, "-4.75", "Coffee Roasters") # same after canonicalization + a = _txn(5, "-4.75", "COFFEE ROASTERS", acct="CHECKING") + b = _txn(5, "-4.75", "Coffee Roasters", acct="CHECKING") out = dedup_transactions([a, b]) assert len(out) == 1 +def test_content_dedup_preserves_transactions_from_different_accounts(): + a = _txn(5, "-4.75", "COFFEE ROASTERS", acct="CHECKING") + b = _txn(5, "-4.75", "Coffee Roasters", acct="SAVINGS") + + out = dedup_transactions([a, b]) + + assert out == [a, b] + + def test_content_dedup_preserves_legit_repeats_within_source(): # Two identical coffees on the same day within ONE statement are legitimate. a = _txn(5, "-4.75", "COFFEE") From 21102b77f7295dc60dd086b56abde5318f91e475 Mon Sep 17 00:00:00 2001 From: Smit Joshi Date: Wed, 12 Aug 2026 16:00:33 +0530 Subject: [PATCH 2/2] test(dedup): cover account-scoped merge --- tests/test_dedup.py | 9 +++++++++ tests/test_normalize_and_cli.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 586ae16..1bae231 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -40,6 +40,15 @@ def test_content_dedup_preserves_transactions_from_different_accounts(): assert out == [a, b] +def test_content_hash_without_account_preserves_legacy_format(): + txn = _txn(5, "-4.75", "Coffee Roasters") + + assert ( + txn.content_hash() + == "029127de86936cfd57927b7d841e0f6e483f407502c92e6a45d538d5dc832903" + ) + + def test_content_dedup_preserves_legit_repeats_within_source(): # Two identical coffees on the same day within ONE statement are legitimate. a = _txn(5, "-4.75", "COFFEE") diff --git a/tests/test_normalize_and_cli.py b/tests/test_normalize_and_cli.py index 0a72763..bcf8ddc 100644 --- a/tests/test_normalize_and_cli.py +++ b/tests/test_normalize_and_cli.py @@ -42,6 +42,28 @@ def test_cross_statement_merge_dedups_overlap(fixture_path): assert len(txns) == 4 +def test_cross_statement_merge_scopes_content_dedup_by_account(): + def mt940(account_id): + return f""":20:STATEMENT +:25:{account_id} +:60F:C240101EUR1000,00 +:61:2401050105D45,20NTRFNONREF +:86:CARD PAYMENT GAS STATION 4471 +:62F:C240131EUR954,80 +- +""" + + txns = normalize_many( + [ + (mt940("ACCOUNT-A"), "account-a-january.mt940"), + (mt940("ACCOUNT-A"), "account-a-overlap.mt940"), + (mt940("ACCOUNT-B"), "account-b-january.mt940"), + ] + ) + + assert [txn.account_id for txn in txns] == ["ACCOUNT-A", "ACCOUNT-B"] + + def test_cli_single_file_json(fixture_path, capsys): rc = main([fixture_path("creditcard.csv"), "--pretty"]) assert rc == 0