Implement Gitea CI and real statement imports
Gitea Actions CI / detect (push) Successful in 3s
Gitea Actions CI / Compose & nginx validation (push) Failing after 20s
Gitea Actions CI / Frontend (lint / type / test / build) (push) Failing after 24s
Gitea Actions CI / Backend (lint / type / test / build) (push) Failing after 1m34s
Gitea Actions CI / detect (push) Successful in 3s
Gitea Actions CI / Compose & nginx validation (push) Failing after 20s
Gitea Actions CI / Frontend (lint / type / test / build) (push) Failing after 24s
Gitea Actions CI / Backend (lint / type / test / build) (push) Failing after 1m34s
This commit is contained in:
@@ -0,0 +1,133 @@
|
||||
"""Hold lifecycle: source bank status and append-only hold resolutions."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0002"
|
||||
down_revision = "0001"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(bind, table: str, column: str) -> bool:
|
||||
inspector = sa.inspect(bind)
|
||||
return column in {item["name"] for item in inspector.get_columns(table)}
|
||||
|
||||
|
||||
def _has_check_constraint(bind, table: str, suffix: str) -> bool:
|
||||
inspector = sa.inspect(bind)
|
||||
return any(
|
||||
(item.get("name") or "").endswith(suffix) for item in inspector.get_check_constraints(table)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
# Revision 0001 materializes the schema from live model metadata, so a
|
||||
# database created fresh already carries these objects; guard every step.
|
||||
if not _has_column(bind, "source_records", "bank_status"):
|
||||
op.add_column(
|
||||
"source_records",
|
||||
sa.Column("bank_status", sa.String(length=8), nullable=False, server_default="posted"),
|
||||
)
|
||||
if bind.dialect.name == "postgresql" and not _has_check_constraint(
|
||||
bind, "source_records", "bank_status_valid"
|
||||
):
|
||||
op.create_check_constraint(
|
||||
"bank_status_valid",
|
||||
"source_records",
|
||||
"bank_status IN ('posted', 'hold')",
|
||||
)
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("hold_resolutions"):
|
||||
op.create_table(
|
||||
"hold_resolutions",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("space_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("hold_source_record_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("resolution", sa.String(length=24), nullable=False),
|
||||
sa.Column("resolving_source_record_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("operation_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("resolved_by_user_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("reason", sa.String(length=500), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')",
|
||||
name="ck_hold_resolutions_resolution_valid",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') "
|
||||
"AND (reason IS NULL OR reason = ''))",
|
||||
name="ck_hold_resolutions_manual_reason_required",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id"],
|
||||
["financial_spaces.id"],
|
||||
ondelete="CASCADE",
|
||||
name="fk_hold_resolutions_space_id_financial_spaces",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id", "hold_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
ondelete="CASCADE",
|
||||
name="fk_hold_resolutions_hold_tenant",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id", "resolving_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
name="fk_hold_resolutions_resolving_tenant",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id", "operation_id"],
|
||||
["operations.space_id", "operations.id"],
|
||||
name="fk_hold_resolutions_operation_tenant",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["resolved_by_user_id"],
|
||||
["users.id"],
|
||||
ondelete="SET NULL",
|
||||
name="fk_hold_resolutions_resolved_by_user_id_users",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_hold_resolutions"),
|
||||
sa.UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"),
|
||||
sa.UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"),
|
||||
)
|
||||
op.create_index("ix_hold_resolutions_space_id", "hold_resolutions", ["space_id"])
|
||||
op.create_index(
|
||||
"ix_hold_resolutions_hold_source_record_id",
|
||||
"hold_resolutions",
|
||||
["hold_source_record_id"],
|
||||
)
|
||||
op.create_index("ix_hold_resolutions_operation_id", "hold_resolutions", ["operation_id"])
|
||||
if bind.dialect.name == "postgresql":
|
||||
# Revision 0001 creates tables from live metadata, so a fresh database
|
||||
# already has hold_resolutions before this revision runs. RLS setup must
|
||||
# therefore be independent of the create-table branch above.
|
||||
op.execute("ALTER TABLE hold_resolutions ENABLE ROW LEVEL SECURITY")
|
||||
op.execute("ALTER TABLE hold_resolutions FORCE ROW LEVEL SECURITY")
|
||||
op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions")
|
||||
op.execute(
|
||||
"CREATE POLICY tenant_isolation ON hold_resolutions "
|
||||
"USING (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid) "
|
||||
"WITH CHECK (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("hold_resolutions"):
|
||||
if bind.dialect.name == "postgresql":
|
||||
op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions")
|
||||
op.drop_index("ix_hold_resolutions_operation_id", table_name="hold_resolutions")
|
||||
op.drop_index("ix_hold_resolutions_hold_source_record_id", table_name="hold_resolutions")
|
||||
op.drop_index("ix_hold_resolutions_space_id", table_name="hold_resolutions")
|
||||
op.drop_table("hold_resolutions")
|
||||
if _has_column(bind, "source_records", "bank_status"):
|
||||
with op.batch_alter_table("source_records") as batch_op:
|
||||
if _has_check_constraint(bind, "source_records", "bank_status_valid"):
|
||||
batch_op.drop_constraint("bank_status_valid", type_="check")
|
||||
batch_op.drop_column("bank_status")
|
||||
@@ -440,7 +440,7 @@ def _close_blockers(
|
||||
uncategorized_count: int,
|
||||
negative_envelopes: list[str],
|
||||
) -> list[dict]:
|
||||
from app.imports.models import StatementDocument
|
||||
from app.imports.models import HoldResolution, SourceRecord, StatementDocument
|
||||
|
||||
blockers = []
|
||||
unresolved = (
|
||||
@@ -455,6 +455,28 @@ def _close_blockers(
|
||||
)
|
||||
if unresolved:
|
||||
blockers.append({"code": "IMPORTS_UNRESOLVED", "count": unresolved})
|
||||
start, end = period_bounds(period.year, period.month)
|
||||
unresolved_holds = (
|
||||
db.query(SourceRecord.id)
|
||||
.join(
|
||||
StatementDocument,
|
||||
(StatementDocument.space_id == SourceRecord.space_id)
|
||||
& (StatementDocument.id == SourceRecord.document_id),
|
||||
)
|
||||
.outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id)
|
||||
.filter(
|
||||
SourceRecord.space_id == space_id,
|
||||
SourceRecord.bank_status == "hold",
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
HoldResolution.id.is_(None),
|
||||
SourceRecord.occurred_on >= start,
|
||||
SourceRecord.occurred_on < end,
|
||||
)
|
||||
.count()
|
||||
)
|
||||
if unresolved_holds:
|
||||
blockers.append({"code": "IMPORT_UNRESOLVED_HOLDS", "count": unresolved_holds})
|
||||
if uncategorized_count:
|
||||
blockers.append({"code": "BUDGET_UNCATEGORIZED_EXPENSES", "count": uncategorized_count})
|
||||
if negative_envelopes:
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter
|
||||
from app.imports.adapters.alfa_xlsx import (
|
||||
AlfaCreditXlsxAdapter as AlfaCreditXlsxAdapter,
|
||||
)
|
||||
from app.imports.adapters.alfa_xlsx import AlfaDebitXlsxAdapter
|
||||
from app.imports.adapters.base import StatementAdapter, UnsupportedStatementLayout
|
||||
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter
|
||||
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter as OzonCreditPdfAdapter
|
||||
from app.imports.adapters.ozon_pdf import OzonDebitPdfAdapter
|
||||
from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter
|
||||
|
||||
# Only the three evidenced-format development contracts are active. The credit
|
||||
# adapters stay present but unregistered until real fixtures exist for them.
|
||||
ADAPTERS: tuple[StatementAdapter, ...] = (
|
||||
AlfaDebitXlsxAdapter(),
|
||||
AlfaCreditXlsxAdapter(),
|
||||
OzonDebitPdfAdapter(),
|
||||
OzonCreditPdfAdapter(),
|
||||
RocketbankDebitPdfAdapter(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from typing import TypedDict
|
||||
|
||||
from openpyxl.workbook import Workbook
|
||||
|
||||
from app.errors import AppError
|
||||
from app.imports.adapters.base import (
|
||||
ALFA_DEBIT_XLSX_VERSION,
|
||||
BANK_STATUS_HOLD,
|
||||
BANK_STATUS_POSTED,
|
||||
DIRECTION_IN,
|
||||
DIRECTION_OUT,
|
||||
PARSER_VERSION,
|
||||
@@ -12,6 +19,7 @@ from app.imports.adapters.base import (
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
amount_to_kopecks,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.readers import open_xlsx
|
||||
|
||||
@@ -198,10 +206,279 @@ def _find_account_value(rows: list[tuple]) -> str | None:
|
||||
return masked
|
||||
|
||||
|
||||
class AlfaDebitXlsxAdapter(AlfaXlsxAdapterBase):
|
||||
class AlfaDebitXlsxAdapter:
|
||||
"""Alfa debit statement contract, evidenced against the real export layout.
|
||||
|
||||
Workbook shape: a 'Выписка по счету' title block, account metadata with
|
||||
the exact statement period ('За период с ... по ...') and header control
|
||||
totals (Поступления / Расходы / Неподтвержденные операции / Текущий
|
||||
баланс), an operation header row (Дата операции, Дата проводки, Код,
|
||||
Категория, Описание, Сумма в валюте счета, Статус), then operation rows
|
||||
terminated by the signature footer. A HOLD posting-date cell marks an
|
||||
immutable hold row; the Код column becomes the external id; a trailing
|
||||
MCC token in the description becomes the row MCC. The raw Расходы control
|
||||
includes hold amounts, so declared_outflow_kopecks is canonicalized to
|
||||
posted-only (raw outflow minus declared hold). Anything else fails closed.
|
||||
"""
|
||||
|
||||
adapter_id = "alfa_debit_xlsx"
|
||||
product = "debit"
|
||||
product_markers = ("дебет", "дебетовая")
|
||||
parser_version = ALFA_DEBIT_XLSX_VERSION
|
||||
bank_code = "alfa"
|
||||
currency = "RUB"
|
||||
|
||||
detect_scan_rows = 60
|
||||
header_scan_rows = 30
|
||||
|
||||
title_label = "выписка по счету"
|
||||
period_re = re.compile(
|
||||
r"за период с\s*(\d{2}\.\d{2}\.\d{4})\s*по\s*(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE
|
||||
)
|
||||
mcc_re = re.compile(r"\bMCC\s*:?\s*(\d{4})\b", re.IGNORECASE)
|
||||
header_labels = {
|
||||
"date": "дата операции",
|
||||
"posted": "дата проводки",
|
||||
"code": "код",
|
||||
"category": "категория",
|
||||
"description": "описание",
|
||||
"status": "статус",
|
||||
}
|
||||
amount_label_prefix = "сумма в"
|
||||
meta_labels = {
|
||||
"account": "номер счета",
|
||||
"currency": "валюта счета",
|
||||
"inflow": "поступления",
|
||||
"outflow": "расходы",
|
||||
"hold": "неподтвержденные операции",
|
||||
"closing": "текущий баланс",
|
||||
}
|
||||
footer_markers = ("подпись сотрудника", "уполномоченное лицо")
|
||||
footer_page_prefix = "страница"
|
||||
hold_token = "hold"
|
||||
|
||||
def detect(self, raw: bytes) -> bool:
|
||||
try:
|
||||
with open_xlsx(raw) as workbook:
|
||||
rows = _debit_read_rows(workbook, self.detect_scan_rows)
|
||||
except Exception:
|
||||
return False
|
||||
cells = [_debit_norm(cell) for row in rows for cell in row]
|
||||
if self.title_label not in cells:
|
||||
return False
|
||||
required = (self.header_labels["date"], self.header_labels["posted"], "статус")
|
||||
return all(label in cells for label in required)
|
||||
|
||||
def parse(self, raw: bytes) -> StatementDraft:
|
||||
with open_xlsx(raw) as workbook:
|
||||
cap = get_settings_row_cap()
|
||||
rows = _debit_read_rows(workbook, cap + 1)
|
||||
if len(rows) > cap:
|
||||
raise UnsupportedStatementLayout("row quota exceeded during parse")
|
||||
header_idx, columns = _debit_find_header(self, rows)
|
||||
meta = _debit_header_metadata(self, rows, header_idx)
|
||||
period_start, period_end = _debit_statement_period(self, rows, header_idx)
|
||||
parsed = _debit_operation_rows(self, rows, header_idx, columns)
|
||||
if not parsed:
|
||||
raise UnsupportedStatementLayout("no operation rows found in Alfa statement")
|
||||
return StatementDraft(
|
||||
adapter_id=self.adapter_id,
|
||||
parser_version=self.parser_version,
|
||||
bank_code=self.bank_code,
|
||||
product=self.product,
|
||||
account_fingerprint=account_fingerprint(self.bank_code, meta["account"]),
|
||||
currency=meta["currency"],
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
opening_balance_kopecks=None,
|
||||
closing_balance_kopecks=meta["closing"],
|
||||
rows=tuple(parsed),
|
||||
declared_inflow_kopecks=meta["inflow"],
|
||||
declared_outflow_kopecks=meta["outflow"],
|
||||
declared_hold_kopecks=meta["hold"],
|
||||
)
|
||||
|
||||
|
||||
def _debit_norm(cell: object) -> str:
|
||||
if cell is None:
|
||||
return ""
|
||||
return str(cell).replace("\u00a0", " ").strip().lower()
|
||||
|
||||
|
||||
def _debit_read_rows(workbook: Workbook, limit: int) -> list[tuple]:
|
||||
sheet = workbook.worksheets[0]
|
||||
rows: list[tuple] = []
|
||||
for index, row in enumerate(sheet.iter_rows(values_only=True)):
|
||||
if index >= limit:
|
||||
break
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _debit_find_header(
|
||||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple]
|
||||
) -> tuple[int, dict[str, int]]:
|
||||
for index, row in enumerate(rows[: adapter.header_scan_rows]):
|
||||
cells = [_debit_norm(cell) for cell in row]
|
||||
columns: dict[str, int] = {}
|
||||
for key, label in adapter.header_labels.items():
|
||||
column = next((i for i, cell in enumerate(cells) if cell == label), None)
|
||||
if column is None:
|
||||
break
|
||||
columns[key] = column
|
||||
if len(columns) != len(adapter.header_labels):
|
||||
continue
|
||||
amount_column = next(
|
||||
(i for i, cell in enumerate(cells) if cell.startswith(adapter.amount_label_prefix)),
|
||||
None,
|
||||
)
|
||||
if amount_column is None:
|
||||
continue
|
||||
columns["amount"] = amount_column
|
||||
return index, columns
|
||||
raise UnsupportedStatementLayout("Alfa operation header row not found")
|
||||
|
||||
|
||||
def _debit_header_value(rows: list[tuple], label: str) -> str:
|
||||
for row in rows:
|
||||
for index, cell in enumerate(row):
|
||||
if _debit_norm(cell) != label:
|
||||
continue
|
||||
for candidate in row[index + 1 :]:
|
||||
text = _cell_str(candidate)
|
||||
if text:
|
||||
return text
|
||||
raise UnsupportedStatementLayout(f"Alfa header metadata not found: {label}")
|
||||
|
||||
|
||||
class _AlfaHeaderMeta(TypedDict):
|
||||
account: str
|
||||
currency: str
|
||||
inflow: int
|
||||
outflow: int
|
||||
hold: int
|
||||
closing: int
|
||||
|
||||
|
||||
def _debit_statement_period(
|
||||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||||
) -> tuple[date, date]:
|
||||
text = "\n".join(_debit_norm(cell) for row in rows[:header_idx] for cell in row)
|
||||
period_match = adapter.period_re.search(text)
|
||||
if period_match is None:
|
||||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||||
period_start = _parse_date(period_match.group(1))
|
||||
period_end = _parse_date(period_match.group(2))
|
||||
if period_start is None or period_end is None:
|
||||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||||
return period_start, period_end
|
||||
|
||||
|
||||
def _debit_header_metadata(
|
||||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||||
) -> _AlfaHeaderMeta:
|
||||
meta: _AlfaHeaderMeta = {
|
||||
"account": _debit_header_value(rows[:header_idx], adapter.meta_labels["account"]),
|
||||
"currency": "RUB",
|
||||
"inflow": 0,
|
||||
"outflow": 0,
|
||||
"hold": 0,
|
||||
"closing": 0,
|
||||
}
|
||||
currency = _debit_header_value(rows[:header_idx], adapter.meta_labels["currency"])
|
||||
if currency.replace(" ", "").upper() not in {"RUB", "RUR"}:
|
||||
raise UnsupportedStatementLayout("unsupported Alfa statement currency")
|
||||
raw_outflow = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["outflow"])
|
||||
)
|
||||
meta["hold"] = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["hold"])
|
||||
)
|
||||
# The raw expense control includes pending (hold) amounts; canonicalize to
|
||||
# posted-only so service reconciliation can compare like with like.
|
||||
canonical_outflow = raw_outflow - meta["hold"]
|
||||
if canonical_outflow < 0:
|
||||
raise UnsupportedStatementLayout("inconsistent Alfa expense header controls")
|
||||
meta["outflow"] = canonical_outflow
|
||||
meta["inflow"] = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["inflow"])
|
||||
)
|
||||
meta["closing"] = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["closing"])
|
||||
)
|
||||
return meta
|
||||
|
||||
|
||||
def _debit_operation_rows(
|
||||
adapter: AlfaDebitXlsxAdapter,
|
||||
rows: list[tuple],
|
||||
header_idx: int,
|
||||
columns: dict[str, int],
|
||||
) -> list[SourceRow]:
|
||||
parsed: list[SourceRow] = []
|
||||
separated_from_operations = False
|
||||
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 2):
|
||||
norm_cells = [_debit_norm(cell) for cell in row]
|
||||
if any(marker in cell for cell in norm_cells for marker in adapter.footer_markers):
|
||||
break
|
||||
if any(cell.startswith(adapter.footer_page_prefix) for cell in norm_cells):
|
||||
break
|
||||
if all(cell == "" for cell in norm_cells):
|
||||
separated_from_operations = bool(parsed)
|
||||
continue
|
||||
occurred = _parse_date(row[columns["date"]])
|
||||
posted_cell = _cell_str(row[columns["posted"]]) or ""
|
||||
status_cell = _cell_str(row[columns["status"]]) or ""
|
||||
bank_status = BANK_STATUS_POSTED
|
||||
if adapter.hold_token in posted_cell.lower() or adapter.hold_token in status_cell.lower():
|
||||
bank_status = BANK_STATUS_HOLD
|
||||
posted_on = None
|
||||
else:
|
||||
posted_on = _parse_date(posted_cell)
|
||||
amount_text = _cell_str(row[columns["amount"]])
|
||||
amount: int | None = None
|
||||
if amount_text is not None:
|
||||
try:
|
||||
amount = abs(money_to_kopecks(amount_text))
|
||||
except UnsupportedStatementLayout:
|
||||
amount = None
|
||||
if occurred is None and amount is None:
|
||||
later_has_operation = any(
|
||||
_parse_date(later[columns["date"]]) is not None
|
||||
for later in rows[offset:]
|
||||
if len(later) > columns["date"]
|
||||
)
|
||||
if separated_from_operations and not later_has_operation:
|
||||
break
|
||||
raise UnsupportedStatementLayout("unexpected row in Alfa operation table")
|
||||
warnings: list[str] = []
|
||||
if occurred is None:
|
||||
warnings.append("occurred_date_missing")
|
||||
if amount is None:
|
||||
amount = 0
|
||||
warnings.append("amount_unparseable")
|
||||
if bank_status == BANK_STATUS_POSTED and posted_on is None and posted_cell:
|
||||
warnings.append("posted_date_unparseable")
|
||||
direction = DIRECTION_OUT if (amount_text or "").startswith("-") else DIRECTION_IN
|
||||
description = _cell_str(row[columns["description"]])
|
||||
mcc_match = adapter.mcc_re.search(description) if description else None
|
||||
parsed.append(
|
||||
SourceRow(
|
||||
external_id=_cell_str(row[columns["code"]]),
|
||||
occurred_on=occurred,
|
||||
posted_on=posted_on,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
description=description,
|
||||
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
|
||||
bank_category=_cell_str(row[columns["category"]]),
|
||||
location=None,
|
||||
row_ref=f"sheet0:row{offset}",
|
||||
warnings=tuple(warnings),
|
||||
bank_status=bank_status,
|
||||
)
|
||||
)
|
||||
separated_from_operations = False
|
||||
return parsed
|
||||
|
||||
|
||||
class AlfaCreditXlsxAdapter(AlfaXlsxAdapterBase):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
@@ -8,7 +9,16 @@ from app.errors import AppError
|
||||
|
||||
DIRECTION_IN = "in"
|
||||
DIRECTION_OUT = "out"
|
||||
BANK_STATUS_POSTED = "posted"
|
||||
BANK_STATUS_HOLD = "hold"
|
||||
BANK_STATUSES = (BANK_STATUS_POSTED, BANK_STATUS_HOLD)
|
||||
# Legacy shared version kept only for the untouched, unregistered credit adapters.
|
||||
PARSER_VERSION = "1"
|
||||
# Per-contract versions: bumped for the rewritten evidenced contracts so old
|
||||
# parse runs can never be confused with the new canonical output.
|
||||
ALFA_DEBIT_XLSX_VERSION = "2"
|
||||
OZON_DEBIT_PDF_VERSION = "2"
|
||||
ROCKETBANK_DEBIT_PDF_VERSION = "2"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -24,6 +34,15 @@ class SourceRow:
|
||||
location: str | None
|
||||
row_ref: str
|
||||
warnings: tuple[str, ...] = ()
|
||||
bank_status: str = BANK_STATUS_POSTED
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.amount_kopecks < 0:
|
||||
raise UnsupportedStatementLayout(
|
||||
"amount_kopecks must be positive; direction carries the sign"
|
||||
)
|
||||
if self.bank_status not in BANK_STATUSES:
|
||||
raise UnsupportedStatementLayout("bank_status must be posted or hold")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -39,6 +58,9 @@ class StatementDraft:
|
||||
opening_balance_kopecks: int | None
|
||||
closing_balance_kopecks: int | None
|
||||
rows: tuple[SourceRow, ...]
|
||||
declared_inflow_kopecks: int | None = None
|
||||
declared_outflow_kopecks: int | None = None
|
||||
declared_hold_kopecks: int | None = None
|
||||
|
||||
|
||||
class UnsupportedStatementLayout(AppError):
|
||||
@@ -77,6 +99,29 @@ def amount_to_kopecks(value: object) -> int:
|
||||
raise UnsupportedStatementLayout(f"unsupported amount cell type: {type(value).__name__}")
|
||||
|
||||
|
||||
def _mask_digits(text: str) -> str:
|
||||
return re.sub(r"\d", "#", text)
|
||||
|
||||
|
||||
CURRENCY_SUFFIX_RE = re.compile(r"(?:₽|RUR|rub|руб\.?)\s*$", re.IGNORECASE)
|
||||
PLAIN_AMOUNT_RE = re.compile(r"^[+-]?\s?\d[\d\s]*(?:[.,]\d{1,2})?$")
|
||||
|
||||
|
||||
def money_to_kopecks(text: object) -> int:
|
||||
"""Parse statement money strings like '-1 234,56 RUR', '+ 500.00 ₽', '12 345'.
|
||||
|
||||
Accepts space/NBSP thousand separators and comma or dot decimal marks.
|
||||
Fails closed with a digit-masked diagnostic; never leaks the raw value.
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
raise UnsupportedStatementLayout(f"unsupported money cell type: {type(text).__name__}")
|
||||
cleaned = text.replace("\u00a0", " ").strip()
|
||||
cleaned = CURRENCY_SUFFIX_RE.sub("", cleaned).strip()
|
||||
if not PLAIN_AMOUNT_RE.fullmatch(cleaned):
|
||||
raise UnsupportedStatementLayout(f"unparseable money amount: {_mask_digits(text)}")
|
||||
return amount_to_kopecks(cleaned)
|
||||
|
||||
|
||||
def account_fingerprint(bank_code: str, masked_identifier: str) -> str:
|
||||
digest = hashlib.sha256(f"{bank_code}|{masked_identifier}".encode()).hexdigest()
|
||||
return digest[:64]
|
||||
|
||||
@@ -6,12 +6,14 @@ from datetime import date, datetime
|
||||
from app.imports.adapters.base import (
|
||||
DIRECTION_IN,
|
||||
DIRECTION_OUT,
|
||||
OZON_DEBIT_PDF_VERSION,
|
||||
PARSER_VERSION,
|
||||
SourceRow,
|
||||
StatementDraft,
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
amount_to_kopecks,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.readers import open_pdf
|
||||
|
||||
@@ -122,10 +124,179 @@ def _first_page_text(raw: bytes) -> str:
|
||||
return pdf.pages[0].extract_text() or ""
|
||||
|
||||
|
||||
class OzonDebitPdfAdapter(OzonPdfAdapterBase):
|
||||
class OzonDebitPdfAdapter:
|
||||
"""Ozon debit statement contract, evidenced against the real document.
|
||||
|
||||
A 'Справка о движении средств' text-layer PDF: a legal-bank header block
|
||||
with account/period/opening balance, per-page tables whose rows keep
|
||||
multiline document IDs and descriptions attached to one transaction, and a
|
||||
closing block with declared inflow/outflow totals and the closing balance.
|
||||
Amounts are signed strings ('- 500.00 ₽'); the sign maps to direction and
|
||||
the stored amount is always positive.
|
||||
"""
|
||||
|
||||
adapter_id = "ozon_debit_pdf"
|
||||
product = "debit"
|
||||
product_markers = ("дебетов",)
|
||||
parser_version = OZON_DEBIT_PDF_VERSION
|
||||
bank_code = "ozon"
|
||||
currency = "RUB"
|
||||
|
||||
bank_marker = "озон банк"
|
||||
title_marker = "справка о движении средств"
|
||||
account_marker_re = re.compile(r"лицевого\s+сч[её]та:?\s*№\s*([\d*•]+)", re.IGNORECASE)
|
||||
period_re = re.compile(
|
||||
r"период\s+выписки:?\s*(\d{2}\.\d{2}\.\d{4})\s*[–—-]\s*(\d{2}\.\d{2}\.\d{4})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
opening_re = re.compile(r"входящий\s+остаток:?\s*(.+)", re.IGNORECASE)
|
||||
inflow_re = re.compile(r"итого\s+зачислений\s+за\s+период:?\s*(.+)", re.IGNORECASE)
|
||||
outflow_re = re.compile(r"итого\s+списаний\s+за\s+период:?\s*(.+)", re.IGNORECASE)
|
||||
closing_re = re.compile(r"исходящий\s+остаток:?\s*(.+)", re.IGNORECASE)
|
||||
amount_re = re.compile(r"^([+-])\s*(\d[\d\s]*(?:[.,]\d{2})?)\s*(?:₽|rur|rub)?$", re.IGNORECASE)
|
||||
table_header_labels = ("дата операции", "документ", "назначение платежа")
|
||||
table_subheader_tokens = ("рубли", "валюта")
|
||||
|
||||
def detect(self, raw: bytes) -> bool:
|
||||
try:
|
||||
text = _first_page_text(raw)
|
||||
except Exception:
|
||||
return False
|
||||
lowered = _ozon_normalize(text)
|
||||
return (
|
||||
self.bank_marker in lowered
|
||||
and self.title_marker in lowered
|
||||
and "лицевого сч" in lowered
|
||||
)
|
||||
|
||||
def parse(self, raw: bytes) -> StatementDraft:
|
||||
pages_text: list[str] = []
|
||||
pages_tables: list[list[list[list[object]]]] = []
|
||||
with open_pdf(raw) as pdf:
|
||||
if not pdf.pages:
|
||||
raise UnsupportedStatementLayout("PDF has no pages")
|
||||
for page_index, page in enumerate(pdf.pages):
|
||||
text = page.extract_text() or ""
|
||||
if page_index == 0 and not text.strip():
|
||||
raise UnsupportedStatementLayout(
|
||||
"PDF has no extractable text layer; OCR is not supported"
|
||||
)
|
||||
pages_text.append(text)
|
||||
extracted = page.extract_tables()
|
||||
pages_tables.append(extracted if extracted else [])
|
||||
return self._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def _draft_from_extracted(
|
||||
self, pages_text: list[str], pages_tables: list[list[list[list[object]]]]
|
||||
) -> StatementDraft:
|
||||
page_zero = _ozon_normalize(pages_text[0])
|
||||
if self.bank_marker not in page_zero or self.title_marker not in page_zero:
|
||||
raise UnsupportedStatementLayout("Ozon statement markers not found")
|
||||
account_match = self.account_marker_re.search(page_zero)
|
||||
if account_match is None:
|
||||
raise UnsupportedStatementLayout("Ozon account line not found")
|
||||
period_match = self.period_re.search(page_zero)
|
||||
if period_match is None:
|
||||
raise UnsupportedStatementLayout("Ozon period line not found")
|
||||
period_start = datetime.strptime(period_match.group(1), "%d.%m.%Y").date()
|
||||
period_end = datetime.strptime(period_match.group(2), "%d.%m.%Y").date()
|
||||
opening = money_to_kopecks(self._required_line(self.opening_re, page_zero, "opening"))
|
||||
full_text = _ozon_normalize("\n".join(pages_text))
|
||||
declared_inflow = money_to_kopecks(self._required_line(self.inflow_re, full_text, "inflow"))
|
||||
declared_outflow = money_to_kopecks(
|
||||
self._required_line(self.outflow_re, full_text, "outflow")
|
||||
)
|
||||
closing = money_to_kopecks(self._required_line(self.closing_re, full_text, "closing"))
|
||||
rows: list[SourceRow] = []
|
||||
for page_index, tables in enumerate(pages_tables, start=1):
|
||||
for table_index, table in enumerate(tables):
|
||||
for row_index, cells in enumerate(table):
|
||||
row = self._table_row(
|
||||
cells, f"page{page_index}:table{table_index}:row{row_index}"
|
||||
)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
if not rows:
|
||||
raise UnsupportedStatementLayout("no statement rows matched the Ozon contract")
|
||||
return StatementDraft(
|
||||
adapter_id=self.adapter_id,
|
||||
parser_version=self.parser_version,
|
||||
bank_code=self.bank_code,
|
||||
product=self.product,
|
||||
account_fingerprint=account_fingerprint(self.bank_code, account_match.group(1)),
|
||||
currency=self.currency,
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
opening_balance_kopecks=opening,
|
||||
closing_balance_kopecks=closing,
|
||||
rows=tuple(rows),
|
||||
declared_inflow_kopecks=declared_inflow,
|
||||
declared_outflow_kopecks=declared_outflow,
|
||||
declared_hold_kopecks=None,
|
||||
)
|
||||
|
||||
def _required_line(self, pattern: re.Pattern[str], text: str, what: str) -> str:
|
||||
match = pattern.search(text)
|
||||
if match is None:
|
||||
raise UnsupportedStatementLayout(f"Ozon {what} total line not found")
|
||||
return match.group(1)
|
||||
|
||||
def _table_row(self, cells: list[object], row_ref: str) -> SourceRow | None:
|
||||
if cells is None:
|
||||
raise UnsupportedStatementLayout("unexpected Ozon table row shape")
|
||||
texts = [_ozon_cell(cell) for cell in cells]
|
||||
if not any(texts):
|
||||
return None
|
||||
lowered = [text.lower() for text in texts]
|
||||
joined = " ".join(text for text in lowered if text)
|
||||
if all(label in joined for label in self.table_header_labels):
|
||||
return None
|
||||
if lowered[0] == "" and any(token in joined for token in self.table_subheader_tokens):
|
||||
return None
|
||||
if lowered[0] == "" or len(texts) < 4:
|
||||
raise UnsupportedStatementLayout("unexpected Ozon table row shape")
|
||||
occurred = None
|
||||
date_match = re.match(r"^(\d{2}\.\d{2}\.\d{4})", texts[0])
|
||||
if date_match is not None:
|
||||
occurred = datetime.strptime(date_match.group(1), "%d.%m.%Y").date()
|
||||
external_id = re.sub(r"\s+", "", texts[1]) if len(texts) > 1 else ""
|
||||
description = " ".join(texts[2].split()) if len(texts) > 2 else ""
|
||||
warnings: list[str] = []
|
||||
if occurred is None:
|
||||
warnings.append("occurred_date_missing")
|
||||
if not external_id:
|
||||
warnings.append("external_id_missing")
|
||||
amount_match = self.amount_re.match(texts[3]) if len(texts) > 3 else None
|
||||
if amount_match is None:
|
||||
amount = 0
|
||||
warnings.append("amount_unparseable")
|
||||
direction = DIRECTION_OUT
|
||||
else:
|
||||
sign, body = amount_match.group(1), amount_match.group(2)
|
||||
amount = amount_to_kopecks(body.replace(" ", "").replace(",", "."))
|
||||
direction = DIRECTION_IN if sign == "+" else DIRECTION_OUT
|
||||
return SourceRow(
|
||||
external_id=external_id or None,
|
||||
occurred_on=occurred,
|
||||
posted_on=None,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
description=description or None,
|
||||
mcc=None,
|
||||
bank_category=None,
|
||||
location=None,
|
||||
row_ref=row_ref,
|
||||
warnings=tuple(warnings),
|
||||
)
|
||||
|
||||
|
||||
def _ozon_normalize(text: str) -> str:
|
||||
return text.replace("\u00a0", " ").lower()
|
||||
|
||||
|
||||
def _ozon_cell(cell: object) -> str:
|
||||
if cell is None:
|
||||
return ""
|
||||
return str(cell).replace("\u00a0", " ").strip()
|
||||
|
||||
|
||||
class OzonCreditPdfAdapter(OzonPdfAdapterBase):
|
||||
|
||||
@@ -1,12 +1,288 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.imports.adapters.ozon_pdf import OzonPdfAdapterBase
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from app.imports.adapters.base import (
|
||||
DIRECTION_IN,
|
||||
DIRECTION_OUT,
|
||||
ROCKETBANK_DEBIT_PDF_VERSION,
|
||||
SourceRow,
|
||||
StatementDraft,
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.readers import open_pdf
|
||||
|
||||
MONTH_NAMES = {
|
||||
"января": 1,
|
||||
"февраля": 2,
|
||||
"марта": 3,
|
||||
"апреля": 4,
|
||||
"мая": 5,
|
||||
"июня": 6,
|
||||
"июля": 7,
|
||||
"августа": 8,
|
||||
"сентября": 9,
|
||||
"октября": 10,
|
||||
"ноября": 11,
|
||||
"декабря": 12,
|
||||
}
|
||||
|
||||
DAY_HEADING_RE = re.compile(
|
||||
r"^(\d{1,2})\s+(января|февраля|марта|апреля|мая|июня|июля|августа|сентября|"
|
||||
r"октября|ноября|декабря)\s+(\d{4})\s+баланс на начало дня:\s*(-?[\d\s]+,\d{2})\s*rur$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
RECORD_RE = re.compile(
|
||||
r"^(\d{1,2}:\d{2})\s+(\d{6,})\s+(.*?)([+-]?\s?[\d\s]+,\d{2})\s*rur$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
HEADER_VALUES_RE = re.compile(
|
||||
r"^(-?[\d\s]+,\d{2})\s*rur\s+(-?[\d\s]+,\d{2})\s*rur(?:\s+(\d{2}\.\d{2}\.\d{4}))?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
PERIOD_RE = re.compile(r"^с\s+(\d{2}\.\d{2}\.\d{4})\s+по\s+(\d{2}\.\d{2}\.\d{4})$", re.IGNORECASE)
|
||||
AMOUNT_RE = re.compile(r"^([+-]?)\s*(\d[\d\s]*,\d{2})$")
|
||||
MCC_RE = re.compile(r"\bmcc\s*:?\s*(\d{4})\b", re.IGNORECASE)
|
||||
LOCATION_RE = re.compile(r"rur,\s*([^,]+?),\s*mcc\b", re.IGNORECASE)
|
||||
|
||||
|
||||
class RocketbankDebitPdfAdapter(OzonPdfAdapterBase):
|
||||
"""Rocketbank debit statement contract: same text-layer line scanning shape."""
|
||||
class RocketbankDebitPdfAdapter:
|
||||
"""Rocketbank debit statement contract, evidenced against the real document.
|
||||
|
||||
Independent text-layer state machine: every page repeats the statement
|
||||
header and column captions; day sections start with a Russian day heading
|
||||
('DD MONTH YYYY БАЛАНС НА НАЧАЛО ДНЯ: X RUR'); each record is a
|
||||
'time + operation code + multiline description + signed RUR amount' line
|
||||
whose description column starts slightly above and continues below that
|
||||
line, so the surrounding non-structural lines are folded into one
|
||||
description. Daily totals and repeated captions are ignored. Statement
|
||||
period, account, opening/closing balances and declared inflow/outflow
|
||||
totals come from the first-page header block.
|
||||
"""
|
||||
|
||||
adapter_id = "rocketbank_debit_pdf"
|
||||
product = "debit"
|
||||
bank_markers = ("рокетбанк", "rocketbank")
|
||||
product_markers = ("дебетов", "дебетовая")
|
||||
parser_version = ROCKETBANK_DEBIT_PDF_VERSION
|
||||
bank_code = "rocketbank"
|
||||
currency = "RUB"
|
||||
|
||||
title_marker = "выписка по счету"
|
||||
card_re = re.compile(r"дебетовая карта рокет\s*(\d{10,30})\s*rur", re.IGNORECASE)
|
||||
column_caption = "время код операции описание сумма"
|
||||
issued_prefix = "дата выдачи выписки:"
|
||||
page_footer_prefix = "страница"
|
||||
daily_total_prefixes = ("всего поступлений:", "всего расходов:")
|
||||
opening_label = "входящий остаток"
|
||||
closing_label = "исходящий остаток"
|
||||
max_fold_lines = 6
|
||||
|
||||
def detect(self, raw: bytes) -> bool:
|
||||
try:
|
||||
with open_pdf(raw) as pdf:
|
||||
if not pdf.pages:
|
||||
return False
|
||||
text = pdf.pages[0].extract_text() or ""
|
||||
except Exception:
|
||||
return False
|
||||
lowered = text.lower()
|
||||
return self.title_marker in lowered and "дебетовая карта рокет" in lowered
|
||||
|
||||
def parse(self, raw: bytes) -> StatementDraft:
|
||||
lines: list[tuple[int, str]] = []
|
||||
with open_pdf(raw) as pdf:
|
||||
if not pdf.pages:
|
||||
raise UnsupportedStatementLayout("PDF has no pages")
|
||||
for page_number, page in enumerate(pdf.pages, start=1):
|
||||
text = page.extract_text() or ""
|
||||
if page_number == 1 and not text.strip():
|
||||
raise UnsupportedStatementLayout(
|
||||
"PDF has no extractable text layer; OCR is not supported"
|
||||
)
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped:
|
||||
lines.append((page_number, stripped))
|
||||
return self._draft_from_lines(lines)
|
||||
|
||||
def _draft_from_lines(self, lines: list[tuple[int, str]]) -> StatementDraft:
|
||||
blob = "\n".join(line.lower() for _, line in lines)
|
||||
if self.title_marker not in blob:
|
||||
raise UnsupportedStatementLayout("Rocketbank statement marker not found")
|
||||
card_match = self.card_re.search(blob)
|
||||
if card_match is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank card line not found")
|
||||
|
||||
period_start: date | None = None
|
||||
period_end: date | None = None
|
||||
opening: int | None = None
|
||||
closing: int | None = None
|
||||
declared_inflow: int | None = None
|
||||
declared_outflow: int | None = None
|
||||
rows: list[SourceRow] = []
|
||||
current_day: date | None = None
|
||||
pending: _PendingRecord | None = None
|
||||
pre_lines: list[tuple[int, str]] = []
|
||||
record_ordinals: dict[int, int] = {}
|
||||
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
page, line = lines[index]
|
||||
low = line.lower()
|
||||
if (
|
||||
low == self.title_marker
|
||||
or low.startswith(self.issued_prefix)
|
||||
or low.startswith(self.page_footer_prefix)
|
||||
or low == self.column_caption
|
||||
or low.startswith(self.daily_total_prefixes)
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
period_match = PERIOD_RE.match(low)
|
||||
if period_match is not None and period_start is None:
|
||||
period_start = _date_from(period_match.group(1))
|
||||
period_end = _date_from(period_match.group(2))
|
||||
index += 1
|
||||
continue
|
||||
if self.opening_label in low or self.closing_label in low:
|
||||
if index + 1 >= len(lines):
|
||||
raise UnsupportedStatementLayout("Rocketbank balance values missing")
|
||||
values_match = HEADER_VALUES_RE.match(lines[index + 1][1].lower())
|
||||
if values_match is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank balance values missing")
|
||||
first = money_to_kopecks(values_match.group(1))
|
||||
second = money_to_kopecks(values_match.group(2))
|
||||
if self.opening_label in low:
|
||||
opening, declared_inflow = first, second
|
||||
else:
|
||||
closing, declared_outflow = first, second
|
||||
index += 2
|
||||
continue
|
||||
day_match = DAY_HEADING_RE.match(low)
|
||||
if day_match is not None:
|
||||
if pending is not None:
|
||||
rows.append(self._record(pending, current_day))
|
||||
pending = None
|
||||
pre_lines = []
|
||||
current_day = date(
|
||||
int(day_match.group(3)),
|
||||
MONTH_NAMES[day_match.group(2).lower()],
|
||||
int(day_match.group(1)),
|
||||
)
|
||||
index += 1
|
||||
continue
|
||||
record_match = RECORD_RE.match(line)
|
||||
if record_match is not None:
|
||||
if pending is not None:
|
||||
rows.append(self._record(pending, current_day))
|
||||
ordinal = record_ordinals.get(page, 0) + 1
|
||||
record_ordinals[page] = ordinal
|
||||
pending = _PendingRecord(
|
||||
row_ref=f"page{page}:record{ordinal}",
|
||||
code=record_match.group(2),
|
||||
description=record_match.group(3).strip(),
|
||||
amount_text=record_match.group(4),
|
||||
pre_lines=pre_lines,
|
||||
)
|
||||
pre_lines = []
|
||||
index += 1
|
||||
continue
|
||||
if pending is not None:
|
||||
pending.post_lines.append((page, line))
|
||||
index += 1
|
||||
continue
|
||||
pre_lines.append((page, line))
|
||||
if len(pre_lines) > self.max_fold_lines:
|
||||
raise UnsupportedStatementLayout("unrecognized content between Rocketbank sections")
|
||||
index += 1
|
||||
if pending is not None:
|
||||
rows.append(self._record(pending, current_day))
|
||||
pre_lines = []
|
||||
if pre_lines:
|
||||
raise UnsupportedStatementLayout("unrecognized trailing Rocketbank content")
|
||||
if (
|
||||
period_start is None
|
||||
or opening is None
|
||||
or closing is None
|
||||
or declared_inflow is None
|
||||
or declared_outflow is None
|
||||
):
|
||||
raise UnsupportedStatementLayout("Rocketbank statement header block not found")
|
||||
if not rows:
|
||||
raise UnsupportedStatementLayout("no Rocketbank operation rows found")
|
||||
return StatementDraft(
|
||||
adapter_id=self.adapter_id,
|
||||
parser_version=self.parser_version,
|
||||
bank_code=self.bank_code,
|
||||
product=self.product,
|
||||
account_fingerprint=account_fingerprint(self.bank_code, card_match.group(1)),
|
||||
currency=self.currency,
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
opening_balance_kopecks=opening,
|
||||
closing_balance_kopecks=closing,
|
||||
rows=tuple(rows),
|
||||
declared_inflow_kopecks=declared_inflow,
|
||||
declared_outflow_kopecks=declared_outflow,
|
||||
declared_hold_kopecks=None,
|
||||
)
|
||||
|
||||
def _record(
|
||||
self,
|
||||
pending: _PendingRecord,
|
||||
current_day: date | None,
|
||||
) -> SourceRow:
|
||||
if current_day is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank record outside a day section")
|
||||
chunks = [text for _, text in pending.pre_lines]
|
||||
chunks.append(pending.description)
|
||||
chunks.extend(text for _, text in pending.post_lines)
|
||||
description = " ".join(" ".join(chunk.split()) for chunk in chunks if chunk.strip())
|
||||
amount_match = AMOUNT_RE.match(" ".join(pending.amount_text.split()))
|
||||
if amount_match is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank record amount missing")
|
||||
sign = amount_match.group(1)
|
||||
amount = money_to_kopecks(amount_match.group(2))
|
||||
direction = DIRECTION_OUT if sign == "-" else DIRECTION_IN
|
||||
mcc_match = MCC_RE.search(description)
|
||||
location_match = LOCATION_RE.search(description)
|
||||
return SourceRow(
|
||||
external_id=pending.code,
|
||||
occurred_on=current_day,
|
||||
posted_on=None,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
description=description or None,
|
||||
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
|
||||
bank_category=None,
|
||||
location=location_match.group(1).strip() if location_match is not None else None,
|
||||
row_ref=pending.row_ref,
|
||||
warnings=(),
|
||||
)
|
||||
|
||||
|
||||
class _PendingRecord:
|
||||
__slots__ = ("row_ref", "code", "description", "amount_text", "pre_lines", "post_lines")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
row_ref: str,
|
||||
code: str,
|
||||
description: str,
|
||||
amount_text: str,
|
||||
pre_lines: list[tuple[int, str]],
|
||||
) -> None:
|
||||
self.row_ref = row_ref
|
||||
self.code = code
|
||||
self.description = description
|
||||
self.amount_text = amount_text
|
||||
self.pre_lines = pre_lines
|
||||
self.post_lines: list[tuple[int, str]] = []
|
||||
|
||||
|
||||
def _date_from(text: str) -> date:
|
||||
day, month, year = (int(part) for part in text.split("."))
|
||||
return date(year, month, day)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import uuid
|
||||
from datetime import date
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
@@ -18,7 +18,7 @@ from sqlalchemy import (
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.db.base import Base, PkMixin, TimestampMixin
|
||||
from app.db.base import Base, PkMixin, TimestampMixin, UTCDateTime, utcnow
|
||||
|
||||
DOCUMENT_STATUSES = (
|
||||
"uploaded",
|
||||
@@ -34,6 +34,9 @@ RUN_STATUSES = ("pending", "running", "succeeded", "failed")
|
||||
CANDIDATE_KINDS = ("duplicate", "transfer")
|
||||
CANDIDATE_CONFIDENCES = ("unique", "possible")
|
||||
CANDIDATE_RESOLUTIONS = ("pending", "is_duplicate", "is_new")
|
||||
BANK_STATUSES = ("posted", "hold")
|
||||
HOLD_RESOLUTIONS = ("matched_posted", "confirmed_as_posted", "discarded_as_cancelled")
|
||||
HOLD_MANUAL_RESOLUTIONS = ("confirmed_as_posted", "discarded_as_cancelled")
|
||||
|
||||
|
||||
class StatementDocument(Base, PkMixin, TimestampMixin):
|
||||
@@ -99,6 +102,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin):
|
||||
|
||||
__tablename__ = "source_records"
|
||||
__table_args__ = (
|
||||
CheckConstraint("bank_status IN ('posted', 'hold')", name="bank_status_valid"),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "document_id"],
|
||||
["statement_documents.space_id", "statement_documents.id"],
|
||||
@@ -128,6 +132,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin):
|
||||
posted_on: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
amount_kopecks: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
direction: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||
bank_status: Mapped[str] = mapped_column(String(8), nullable=False, default="posted")
|
||||
description: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
mcc: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
bank_category: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||
@@ -209,3 +214,57 @@ class DedupCandidate(Base, PkMixin, TimestampMixin):
|
||||
decided_by_user_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
|
||||
|
||||
class HoldResolution(Base, PkMixin, TimestampMixin):
|
||||
"""Append-only, tenant-scoped settlement of one HOLD source record.
|
||||
|
||||
The extracted HOLD row itself is never mutated. At most one resolution
|
||||
exists per hold source record; automatic matches (`matched_posted`) are
|
||||
created only when a later statement is confirmed, and manual actions
|
||||
(`confirmed_as_posted`, `discarded_as_cancelled`) always carry a reason.
|
||||
"""
|
||||
|
||||
__tablename__ = "hold_resolutions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')",
|
||||
name="resolution_valid",
|
||||
),
|
||||
CheckConstraint(
|
||||
"NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') "
|
||||
"AND (reason IS NULL OR reason = ''))",
|
||||
name="manual_reason_required",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "hold_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
ondelete="CASCADE",
|
||||
name="fk_hold_resolutions_hold_tenant",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "resolving_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
name="fk_hold_resolutions_resolving_tenant",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "operation_id"],
|
||||
["operations.space_id", "operations.id"],
|
||||
name="fk_hold_resolutions_operation_tenant",
|
||||
),
|
||||
UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"),
|
||||
UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"),
|
||||
)
|
||||
|
||||
space_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("financial_spaces.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
hold_source_record_id: Mapped[uuid.UUID] = mapped_column(Uuid, nullable=False, index=True)
|
||||
resolution: Mapped[str] = mapped_column(String(24), nullable=False)
|
||||
resolving_source_record_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True)
|
||||
operation_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True, index=True)
|
||||
resolved_by_user_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
reason: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
resolved_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False, default=utcnow)
|
||||
|
||||
@@ -33,6 +33,14 @@ class ConfirmBody(BaseModel):
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
class HoldResolveBody(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
action: str
|
||||
reason: str
|
||||
account_id: uuid.UUID | None = Field(default=None, alias="accountId")
|
||||
|
||||
|
||||
def summary(document: Any) -> dict:
|
||||
return {
|
||||
"id": document.id,
|
||||
@@ -100,21 +108,34 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
document = service._document(db, space_id=context.space.id, document_id=document_id)
|
||||
base = summary(document)
|
||||
reconciliation = raw.get("reconciliation") or {}
|
||||
rows = [
|
||||
{
|
||||
"id": item["id"],
|
||||
"externalId": item["external_id"],
|
||||
"occurredOn": item["occurred_on"],
|
||||
"amountKopecks": item["amount_kopecks"],
|
||||
"direction": item["direction"],
|
||||
"description": item["description"],
|
||||
"payeeHint": None,
|
||||
"warnings": (item["warnings"] or {}).get("codes", []),
|
||||
"decision": "pending" if item["critical"] else item["review_decision"],
|
||||
"duplicateOfOperationId": item["duplicate_operation_id"],
|
||||
}
|
||||
for item in raw["records"]
|
||||
]
|
||||
rows = []
|
||||
for item in raw["records"]:
|
||||
hold = item.get("hold_resolution")
|
||||
rows.append(
|
||||
{
|
||||
"id": item["id"],
|
||||
"externalId": item["external_id"],
|
||||
"occurredOn": item["occurred_on"],
|
||||
"amountKopecks": item["amount_kopecks"],
|
||||
"direction": item["direction"],
|
||||
"bankStatus": item.get("bank_status", "posted"),
|
||||
"description": item["description"],
|
||||
"payeeHint": None,
|
||||
"warnings": (item["warnings"] or {}).get("codes", []),
|
||||
"decision": "pending" if item["critical"] else item["review_decision"],
|
||||
"duplicateOfOperationId": item["duplicate_operation_id"],
|
||||
"holdResolution": None
|
||||
if hold is None
|
||||
else {
|
||||
"resolution": hold["resolution"],
|
||||
"resolvingSourceRecordId": hold["resolving_source_record_id"],
|
||||
"operationId": hold["operation_id"],
|
||||
"resolvedByUserId": hold["resolved_by_user_id"],
|
||||
"reason": hold["reason"],
|
||||
"resolvedAt": hold["resolved_at"],
|
||||
},
|
||||
}
|
||||
)
|
||||
blockers = [
|
||||
{"code": "IMPORT_FIELD_UNRESOLVED", "message": None, "rowId": str(item["id"])}
|
||||
for item in raw["records"]
|
||||
@@ -136,7 +157,16 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
"status": status,
|
||||
"openingBalanceKopecks": reconciliation.get("opening_balance_kopecks"),
|
||||
"closingBalanceKopecks": reconciliation.get("closing_balance_kopecks"),
|
||||
"computedClosingKopecks": None,
|
||||
"computedClosingKopecks": reconciliation.get("computed_closing_kopecks"),
|
||||
"declaredInflowKopecks": reconciliation.get("declared_inflow_kopecks"),
|
||||
"declaredOutflowKopecks": reconciliation.get("declared_outflow_kopecks"),
|
||||
"declaredHoldKopecks": reconciliation.get("declared_hold_kopecks"),
|
||||
"parsedPostedInflowKopecks": reconciliation.get("parsed_posted_inflow_kopecks"),
|
||||
"parsedPostedOutflowKopecks": reconciliation.get("parsed_posted_outflow_kopecks"),
|
||||
"parsedHoldTotalKopecks": reconciliation.get("parsed_hold_total_kopecks"),
|
||||
"postedTotalsMatched": reconciliation.get("posted_totals_matched"),
|
||||
"holdTotalsMatched": reconciliation.get("hold_totals_matched"),
|
||||
"balanceMatched": reconciliation.get("balance_matched"),
|
||||
"reason": reconciliation.get("reason"),
|
||||
},
|
||||
"rows": rows,
|
||||
@@ -181,6 +211,26 @@ def decide_row(
|
||||
return detail(document_id, db, context)
|
||||
|
||||
|
||||
@router.post("/{document_id}/rows/{row_id}/resolve")
|
||||
def resolve_hold(
|
||||
document_id: uuid.UUID,
|
||||
row_id: uuid.UUID,
|
||||
body: HoldResolveBody,
|
||||
db: Db,
|
||||
context: Context,
|
||||
_csrf: Csrf,
|
||||
) -> dict:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=context.space.id,
|
||||
document_id=document_id,
|
||||
source_record_id=row_id,
|
||||
user=context.user,
|
||||
**body.model_dump(by_alias=False),
|
||||
)
|
||||
return detail(document_id, db, context)
|
||||
|
||||
|
||||
@router.post("/{document_id}/confirm")
|
||||
def confirm(
|
||||
document_id: uuid.UUID,
|
||||
|
||||
+330
-21
@@ -9,8 +9,9 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.audit import service as audit
|
||||
from app.config import get_settings
|
||||
from app.db.base import utcnow
|
||||
from app.db.session import session_scope
|
||||
from app.errors import AppError, conflict, not_found
|
||||
from app.errors import AppError, bad_request, conflict, not_found
|
||||
from app.identity.models import User
|
||||
from app.imports.adapters import detect_adapter, magic_kind
|
||||
from app.imports.adapters.base import SourceRow, StatementDraft, record_fingerprint
|
||||
@@ -18,6 +19,7 @@ from app.imports.blob import get_blob_store, new_object_key
|
||||
from app.imports.crypto import decrypt_file, encrypt_file
|
||||
from app.imports.models import (
|
||||
DedupCandidate,
|
||||
HoldResolution,
|
||||
ParseRun,
|
||||
SourceRecord,
|
||||
SourceRecordLink,
|
||||
@@ -31,6 +33,8 @@ from app.ledger.service import (
|
||||
ensure_period_open_for_date,
|
||||
)
|
||||
|
||||
HOLD_MANUAL_ACTIONS = ("confirm_as_posted", "discard_as_cancelled")
|
||||
|
||||
|
||||
def _document(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID) -> StatementDocument:
|
||||
document = (
|
||||
@@ -128,12 +132,16 @@ def _draft_json(draft: StatementDraft) -> dict[str, Any]:
|
||||
def _source_from_row(
|
||||
*, space_id: uuid.UUID, document_id: uuid.UUID, run_id: uuid.UUID, row: SourceRow
|
||||
) -> SourceRecord:
|
||||
critical_warnings = {"occurred_date_missing", "amount_unparseable"}
|
||||
critical_warnings = {"occurred_date_missing", "amount_unparseable", "bank_status_invalid"}
|
||||
warnings = list(row.warnings)
|
||||
if row.direction not in {"in", "out"}:
|
||||
warnings.append("direction_invalid")
|
||||
if row.amount_kopecks == 0:
|
||||
warnings.append("amount_zero")
|
||||
bank_status = getattr(row, "bank_status", "posted")
|
||||
if bank_status not in {"posted", "hold"}:
|
||||
warnings.append("bank_status_invalid")
|
||||
bank_status = "posted"
|
||||
return SourceRecord(
|
||||
space_id=space_id,
|
||||
document_id=document_id,
|
||||
@@ -143,6 +151,7 @@ def _source_from_row(
|
||||
posted_on=row.posted_on,
|
||||
amount_kopecks=abs(row.amount_kopecks),
|
||||
direction=row.direction,
|
||||
bank_status=bank_status,
|
||||
description=row.description,
|
||||
mcc=row.mcc,
|
||||
bank_category=row.bank_category,
|
||||
@@ -158,9 +167,66 @@ def _source_from_row(
|
||||
)
|
||||
|
||||
|
||||
def _reconciliation_summary(draft: Any) -> dict[str, Any]:
|
||||
"""Reconcile parsed rows against declared bank controls with stable diagnostics.
|
||||
|
||||
Posted rows are compared with the declared inflow/outflow controls, HOLD
|
||||
rows with the declared pending total, and where both balances exist the
|
||||
balance equation must hold. Every check keeps an explicit tri-state field:
|
||||
True (matched), False (mismatch, review-blocking), None (not declared).
|
||||
"""
|
||||
rows = list(draft.rows)
|
||||
posted = [row for row in rows if getattr(row, "bank_status", "posted") == "posted"]
|
||||
holds = [row for row in rows if getattr(row, "bank_status", "posted") == "hold"]
|
||||
posted_inflow = sum(row.amount_kopecks for row in posted if row.direction == "in")
|
||||
posted_outflow = sum(row.amount_kopecks for row in posted if row.direction == "out")
|
||||
hold_total = sum(abs(row.amount_kopecks) for row in holds)
|
||||
declared_inflow = getattr(draft, "declared_inflow_kopecks", None)
|
||||
declared_outflow = getattr(draft, "declared_outflow_kopecks", None)
|
||||
declared_hold = getattr(draft, "declared_hold_kopecks", None)
|
||||
full = draft.opening_balance_kopecks is not None and draft.closing_balance_kopecks is not None
|
||||
checks: list[bool] = []
|
||||
balance_matched: bool | None = None
|
||||
computed_closing: int | None = None
|
||||
if full:
|
||||
computed_closing = int(draft.opening_balance_kopecks) + posted_inflow - posted_outflow
|
||||
balance_matched = computed_closing == int(draft.closing_balance_kopecks)
|
||||
checks.append(balance_matched)
|
||||
posted_totals_matched: bool | None = None
|
||||
if declared_inflow is not None or declared_outflow is not None:
|
||||
posted_totals_matched = (declared_inflow is None or declared_inflow == posted_inflow) and (
|
||||
declared_outflow is None or declared_outflow == posted_outflow
|
||||
)
|
||||
checks.append(posted_totals_matched)
|
||||
hold_totals_matched: bool | None = None
|
||||
if declared_hold is not None:
|
||||
hold_totals_matched = declared_hold == hold_total
|
||||
checks.append(hold_totals_matched)
|
||||
return {
|
||||
"currency": draft.currency,
|
||||
"opening_balance_kopecks": draft.opening_balance_kopecks,
|
||||
"closing_balance_kopecks": draft.closing_balance_kopecks,
|
||||
"full": full,
|
||||
"declared_inflow_kopecks": declared_inflow,
|
||||
"declared_outflow_kopecks": declared_outflow,
|
||||
"declared_hold_kopecks": declared_hold,
|
||||
"parsed_posted_inflow_kopecks": posted_inflow,
|
||||
"parsed_posted_outflow_kopecks": posted_outflow,
|
||||
"parsed_hold_total_kopecks": hold_total,
|
||||
"posted_totals_matched": posted_totals_matched,
|
||||
"hold_totals_matched": hold_totals_matched,
|
||||
"balance_matched": balance_matched,
|
||||
"computed_closing_kopecks": computed_closing,
|
||||
"matched": all(checks) if checks else None,
|
||||
"confirmed_without_full_reconciliation": False,
|
||||
}
|
||||
|
||||
|
||||
def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun) -> bool:
|
||||
pending = False
|
||||
for source in db.query(SourceRecord).filter_by(parse_run_id=run.id).all():
|
||||
if source.bank_status == "hold":
|
||||
continue
|
||||
duplicate = None
|
||||
if source.external_id:
|
||||
matches = (
|
||||
@@ -170,6 +236,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.external_id == source.external_id,
|
||||
SourceRecord.bank_status == "posted",
|
||||
ParseRun.adapter_id == run.adapter_id,
|
||||
)
|
||||
.all()
|
||||
@@ -183,6 +250,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.record_fp == source.record_fp,
|
||||
SourceRecord.direction == source.direction,
|
||||
SourceRecord.bank_status == "posted",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
@@ -212,6 +280,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.amount_kopecks == source.amount_kopecks,
|
||||
SourceRecord.direction == opposite,
|
||||
SourceRecord.bank_status == "posted",
|
||||
SourceRecord.occurred_on >= source.occurred_on - timedelta(days=1),
|
||||
SourceRecord.occurred_on <= source.occurred_on + timedelta(days=1),
|
||||
)
|
||||
@@ -234,6 +303,105 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
return pending
|
||||
|
||||
|
||||
def _match_hold_resolutions(
|
||||
db: Session, *, document: StatementDocument, run: ParseRun
|
||||
) -> dict[uuid.UUID, uuid.UUID]:
|
||||
"""Conservatively match unresolved confirmed-doc HOLDs to new posted records.
|
||||
|
||||
Candidate pairs are searched only within the same space, mapped account,
|
||||
and statement contract (adapter). A stable external ID wins; otherwise a
|
||||
single unambiguous date/amount/direction/normalized-description
|
||||
fingerprint match is required. Any overlap ambiguity leaves every
|
||||
involved HOLD unresolved (review-required). Resolutions are written only
|
||||
when the new statement is confirmed.
|
||||
"""
|
||||
if run.adapter_id is None or document.account_id is None:
|
||||
return {}
|
||||
holds = (
|
||||
db.query(SourceRecord)
|
||||
.join(
|
||||
StatementDocument,
|
||||
(StatementDocument.space_id == SourceRecord.space_id)
|
||||
& (StatementDocument.id == SourceRecord.document_id),
|
||||
)
|
||||
.outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id)
|
||||
.filter(
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.bank_status == "hold",
|
||||
SourceRecord.parse_run_id != run.id,
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
StatementDocument.adapter_id == run.adapter_id,
|
||||
StatementDocument.account_id == document.account_id,
|
||||
HoldResolution.id.is_(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
if not holds:
|
||||
return {}
|
||||
posted = (
|
||||
db.query(SourceRecord)
|
||||
.filter_by(parse_run_id=run.id, bank_status="posted")
|
||||
.filter(SourceRecord.review_decision != "discard")
|
||||
.all()
|
||||
)
|
||||
by_external: dict[str, list[SourceRecord]] = {}
|
||||
by_fingerprint: dict[tuple[str, str], list[SourceRecord]] = {}
|
||||
for record in posted:
|
||||
if record.external_id:
|
||||
by_external.setdefault(record.external_id, []).append(record)
|
||||
by_fingerprint.setdefault((record.record_fp, record.direction), []).append(record)
|
||||
candidates: dict[uuid.UUID, set[uuid.UUID]] = {}
|
||||
for hold in holds:
|
||||
matched: set[uuid.UUID] = set()
|
||||
if hold.external_id:
|
||||
matched.update(item.id for item in by_external.get(hold.external_id, []))
|
||||
matched.update(item.id for item in by_fingerprint.get((hold.record_fp, hold.direction), []))
|
||||
if len(matched) == 1:
|
||||
candidates[hold.id] = matched
|
||||
claimed: dict[uuid.UUID, int] = {}
|
||||
for posted_ids in candidates.values():
|
||||
for posted_id in posted_ids:
|
||||
claimed[posted_id] = claimed.get(posted_id, 0) + 1
|
||||
return {
|
||||
hold_id: next(iter(posted_ids))
|
||||
for hold_id, posted_ids in candidates.items()
|
||||
if all(claimed[posted_id] == 1 for posted_id in posted_ids)
|
||||
}
|
||||
|
||||
|
||||
def _apply_hold_matches(
|
||||
db: Session,
|
||||
*,
|
||||
document: StatementDocument,
|
||||
run: ParseRun,
|
||||
user: User,
|
||||
matches: dict[uuid.UUID, uuid.UUID],
|
||||
) -> None:
|
||||
for hold_id, posted_id in matches.items():
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=document.space_id,
|
||||
hold_source_record_id=hold_id,
|
||||
resolution="matched_posted",
|
||||
resolving_source_record_id=posted_id,
|
||||
resolved_at=utcnow(),
|
||||
)
|
||||
)
|
||||
if matches:
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
event_type="import.holds_matched_posted",
|
||||
space_id=document.space_id,
|
||||
user_id=user.id,
|
||||
detail={
|
||||
"document_id": str(document.id),
|
||||
"holds": sorted(str(hold_id) for hold_id in matches),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> str:
|
||||
with session_scope(space_id) as db:
|
||||
document = _document(db, space_id=space_id, document_id=document_id)
|
||||
@@ -273,23 +441,8 @@ def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> s
|
||||
document.adapter_id = adapter.adapter_id
|
||||
document.account_fingerprint = draft.account_fingerprint
|
||||
document.active_parse_run_id = run.id
|
||||
document.reconciliation = {
|
||||
"currency": draft.currency,
|
||||
"opening_balance_kopecks": draft.opening_balance_kopecks,
|
||||
"closing_balance_kopecks": draft.closing_balance_kopecks,
|
||||
"full": draft.opening_balance_kopecks is not None
|
||||
and draft.closing_balance_kopecks is not None,
|
||||
"confirmed_without_full_reconciliation": False,
|
||||
}
|
||||
if document.reconciliation["full"]:
|
||||
signed_total = sum(
|
||||
row.amount_kopecks if row.direction == "in" else -row.amount_kopecks
|
||||
for row in draft.rows
|
||||
)
|
||||
computed = int(draft.opening_balance_kopecks or 0) + signed_total
|
||||
document.reconciliation["computed_closing_kopecks"] = computed
|
||||
document.reconciliation["matched"] = computed == draft.closing_balance_kopecks
|
||||
if draft.account_fingerprint and document.account_id is None:
|
||||
document.reconciliation = _reconciliation_summary(draft)
|
||||
if document.account_id is None and draft.account_fingerprint:
|
||||
matches = (
|
||||
db.query(Account)
|
||||
.filter_by(space_id=space_id, fingerprint=draft.account_fingerprint)
|
||||
@@ -351,6 +504,25 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
.order_by(SourceRecord.created_at)
|
||||
.all()
|
||||
)
|
||||
hold_ids = [record.id for record in records]
|
||||
resolutions: dict[uuid.UUID, dict[str, Any]] = {}
|
||||
if hold_ids:
|
||||
for item in (
|
||||
db.query(HoldResolution)
|
||||
.filter(
|
||||
HoldResolution.space_id == space_id,
|
||||
HoldResolution.hold_source_record_id.in_(hold_ids),
|
||||
)
|
||||
.all()
|
||||
):
|
||||
resolutions[item.hold_source_record_id] = {
|
||||
"resolution": item.resolution,
|
||||
"resolving_source_record_id": item.resolving_source_record_id,
|
||||
"operation_id": item.operation_id,
|
||||
"resolved_by_user_id": item.resolved_by_user_id,
|
||||
"reason": item.reason,
|
||||
"resolved_at": item.resolved_at,
|
||||
}
|
||||
return {
|
||||
"id": document.id,
|
||||
"status": document.status,
|
||||
@@ -367,11 +539,13 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
"posted_on": item.posted_on,
|
||||
"amount_kopecks": item.amount_kopecks,
|
||||
"direction": item.direction,
|
||||
"bank_status": item.bank_status,
|
||||
"description": item.description,
|
||||
"critical": item.critical,
|
||||
"warnings": item.field_warnings,
|
||||
"review_decision": item.review_decision,
|
||||
"duplicate_operation_id": item.duplicate_operation_id,
|
||||
"hold_resolution": resolutions.get(item.id),
|
||||
}
|
||||
for item in records
|
||||
],
|
||||
@@ -489,6 +663,119 @@ def set_review_confirmation(
|
||||
return document
|
||||
|
||||
|
||||
def resolve_hold(
|
||||
db: Session,
|
||||
*,
|
||||
space_id: uuid.UUID,
|
||||
document_id: uuid.UUID,
|
||||
source_record_id: uuid.UUID,
|
||||
user: User,
|
||||
action: str,
|
||||
reason: str | None,
|
||||
account_id: uuid.UUID | None = None,
|
||||
) -> HoldResolution:
|
||||
"""Manually settle one HOLD: book it as a normal operation or discard it."""
|
||||
document = _document(db, space_id=space_id, document_id=document_id)
|
||||
if document.status != "confirmed":
|
||||
raise conflict(
|
||||
"IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED",
|
||||
"Holds can only be resolved after the import is confirmed",
|
||||
)
|
||||
if action not in HOLD_MANUAL_ACTIONS:
|
||||
raise bad_request("IMPORT_HOLD_ACTION_INVALID", "Unknown hold resolution action")
|
||||
if reason is None or not reason.strip():
|
||||
raise conflict("IMPORT_HOLD_REASON_REQUIRED", "A resolution reason is required")
|
||||
source = (
|
||||
db.query(SourceRecord)
|
||||
.filter_by(
|
||||
id=source_record_id,
|
||||
space_id=space_id,
|
||||
parse_run_id=document.active_parse_run_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if source is None:
|
||||
raise not_found("IMPORT_HOLD_NOT_FOUND", "Import row not found")
|
||||
if source.bank_status != "hold":
|
||||
raise conflict("IMPORT_HOLD_INVALID", "Only hold rows can be resolved manually")
|
||||
existing = db.query(HoldResolution.id).filter_by(hold_source_record_id=source.id).one_or_none()
|
||||
if existing is not None:
|
||||
raise conflict("IMPORT_HOLD_ALREADY_RESOLVED", "Hold is already resolved")
|
||||
if source.occurred_on is None:
|
||||
raise conflict("IMPORT_HOLD_DATE_MISSING", "Hold has no usable operation date")
|
||||
if source.direction not in {"in", "out"}:
|
||||
raise conflict("IMPORT_HOLD_DIRECTION_INVALID", "Hold direction is invalid")
|
||||
resolution = HoldResolution(
|
||||
space_id=space_id,
|
||||
hold_source_record_id=source.id,
|
||||
resolution="confirmed_as_posted"
|
||||
if action == "confirm_as_posted"
|
||||
else "discarded_as_cancelled",
|
||||
resolved_by_user_id=user.id,
|
||||
reason=reason.strip(),
|
||||
resolved_at=utcnow(),
|
||||
)
|
||||
if action == "confirm_as_posted":
|
||||
mapped_account_id = account_id or document.account_id
|
||||
if mapped_account_id is None:
|
||||
raise conflict(
|
||||
"IMPORT_HOLD_ACCOUNT_REQUIRED",
|
||||
"A mapped account is required to confirm a hold as posted",
|
||||
)
|
||||
account = db.query(Account).filter_by(id=mapped_account_id, space_id=space_id).one_or_none()
|
||||
if account is None:
|
||||
raise not_found("LEDGER_ACCOUNT_NOT_FOUND", "Account not found")
|
||||
if source.direction == "in":
|
||||
operation = create_income(
|
||||
db,
|
||||
space_id=space_id,
|
||||
user=user,
|
||||
account_id=account.id,
|
||||
amount_kopecks=source.amount_kopecks,
|
||||
occurred_on=source.occurred_on,
|
||||
posted_on=source.posted_on,
|
||||
description=source.description,
|
||||
request_key=f"import:{source.id}",
|
||||
)
|
||||
else:
|
||||
operation = create_expense(
|
||||
db,
|
||||
space_id=space_id,
|
||||
user=user,
|
||||
account_id=account.id,
|
||||
amount_kopecks=source.amount_kopecks,
|
||||
occurred_on=source.occurred_on,
|
||||
posted_on=source.posted_on,
|
||||
description=source.description,
|
||||
request_key=f"import:{source.id}",
|
||||
)
|
||||
db.add(
|
||||
SourceRecordLink(
|
||||
space_id=space_id,
|
||||
source_record_id=source.id,
|
||||
operation_id=operation.id,
|
||||
kind="hold_confirmed",
|
||||
)
|
||||
)
|
||||
resolution.operation_id = operation.id
|
||||
db.add(resolution)
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
event_type="import.hold_confirmed_as_posted"
|
||||
if action == "confirm_as_posted"
|
||||
else "import.hold_discarded_as_cancelled",
|
||||
space_id=space_id,
|
||||
user_id=user.id,
|
||||
detail={
|
||||
"document_id": str(document.id),
|
||||
"source_record_id": str(source.id),
|
||||
"reason": reason.strip(),
|
||||
},
|
||||
)
|
||||
return resolution
|
||||
|
||||
|
||||
def confirm_document(
|
||||
db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID, user: User
|
||||
) -> StatementDocument:
|
||||
@@ -503,7 +790,7 @@ def confirm_document(
|
||||
"confirmed_without_full_reconciliation"
|
||||
):
|
||||
raise conflict("IMPORT_RECONCILIATION_REQUIRED", "Reconciliation must be confirmed")
|
||||
if reconciliation.get("full") and reconciliation.get("matched") is False:
|
||||
if reconciliation.get("matched") is False:
|
||||
raise conflict("IMPORT_RECONCILIATION_MISMATCH", "Statement arithmetic does not reconcile")
|
||||
pending = (
|
||||
db.query(DedupCandidate.id)
|
||||
@@ -519,7 +806,18 @@ def confirm_document(
|
||||
records = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).all()
|
||||
if any(item.critical for item in records):
|
||||
raise conflict("IMPORT_FIELDS_UNRESOLVED", "Critical source fields remain unresolved")
|
||||
run = db.get(ParseRun, document.active_parse_run_id)
|
||||
if run is not None:
|
||||
_apply_hold_matches(
|
||||
db,
|
||||
document=document,
|
||||
run=run,
|
||||
user=user,
|
||||
matches=_match_hold_resolutions(db, document=document, run=run),
|
||||
)
|
||||
for source in records:
|
||||
if source.bank_status == "hold":
|
||||
continue
|
||||
if source.review_decision == "discard":
|
||||
continue
|
||||
if source.review_decision == "transfer":
|
||||
@@ -755,10 +1053,21 @@ def revert_document(
|
||||
document = _document(db, space_id=space_id, document_id=document_id)
|
||||
if document.status != "confirmed":
|
||||
return document
|
||||
if document.active_parse_run_id:
|
||||
db.query(HoldResolution).filter(
|
||||
HoldResolution.space_id == space_id,
|
||||
HoldResolution.resolution == "matched_posted",
|
||||
HoldResolution.resolving_source_record_id.in_(
|
||||
db.query(SourceRecord.id).filter_by(parse_run_id=document.active_parse_run_id)
|
||||
),
|
||||
).delete(synchronize_session=False)
|
||||
links = (
|
||||
db.query(SourceRecordLink)
|
||||
.join(SourceRecord, SourceRecord.id == SourceRecordLink.source_record_id)
|
||||
.filter(SourceRecord.parse_run_id == document.active_parse_run_id)
|
||||
.filter(
|
||||
SourceRecord.parse_run_id == document.active_parse_run_id,
|
||||
SourceRecordLink.kind != "hold_confirmed",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
transfer_candidates = (
|
||||
|
||||
@@ -39,6 +39,10 @@ package = false
|
||||
pythonpath = ["."]
|
||||
testpaths = ["tests"]
|
||||
filterwarnings = ["ignore::DeprecationWarning"]
|
||||
addopts = "-m 'not private_fixtures'"
|
||||
markers = [
|
||||
"private_fixtures: real-statement private suite; run explicitly with -m private_fixtures (never in CI)",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
|
||||
@@ -0,0 +1,825 @@
|
||||
"""Sanitized synthetic contract tests for the evidenced statement adapters.
|
||||
|
||||
These tests never touch real statements: every input below is synthetic and
|
||||
tracked. Real-file validation lives in the separately collected private
|
||||
suite (tests/test_private_fixtures.py, marker `private_fixtures`).
|
||||
"""
|
||||
|
||||
import io
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
from openpyxl import Workbook
|
||||
|
||||
from app.errors import AppError
|
||||
from app.imports.adapters import ADAPTERS, detect_adapter, magic_kind
|
||||
from app.imports.adapters import alfa_xlsx as alfa_module
|
||||
from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter
|
||||
from app.imports.adapters.base import (
|
||||
ALFA_DEBIT_XLSX_VERSION,
|
||||
BANK_STATUS_HOLD,
|
||||
OZON_DEBIT_PDF_VERSION,
|
||||
PARSER_VERSION,
|
||||
ROCKETBANK_DEBIT_PDF_VERSION,
|
||||
SourceRow,
|
||||
StatementDraft,
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter
|
||||
from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter
|
||||
from app.imports.service import _reconciliation_summary
|
||||
|
||||
NBSP = "\u00a0"
|
||||
|
||||
|
||||
def make_source_row(**overrides: object) -> SourceRow:
|
||||
values: dict[str, object] = {
|
||||
"external_id": None,
|
||||
"occurred_on": date(2026, 1, 1),
|
||||
"posted_on": date(2026, 1, 2),
|
||||
"amount_kopecks": 100,
|
||||
"direction": "out",
|
||||
"description": None,
|
||||
"mcc": None,
|
||||
"bank_category": None,
|
||||
"location": None,
|
||||
"row_ref": "row1",
|
||||
}
|
||||
values.update(overrides)
|
||||
return SourceRow(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_draft(**overrides: object) -> StatementDraft:
|
||||
values: dict[str, object] = {
|
||||
"adapter_id": "alfa_debit_xlsx",
|
||||
"parser_version": ALFA_DEBIT_XLSX_VERSION,
|
||||
"bank_code": "alfa",
|
||||
"product": "debit",
|
||||
"account_fingerprint": None,
|
||||
"currency": "RUB",
|
||||
"period_start": None,
|
||||
"period_end": None,
|
||||
"opening_balance_kopecks": None,
|
||||
"closing_balance_kopecks": None,
|
||||
"rows": (),
|
||||
}
|
||||
values.update(overrides)
|
||||
return StatementDraft(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestBaseContract:
|
||||
def test_source_row_defaults_to_posted(self) -> None:
|
||||
assert make_source_row().bank_status == "posted"
|
||||
|
||||
def test_negative_amount_is_rejected(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
make_source_row(amount_kopecks=-1)
|
||||
|
||||
def test_unknown_bank_status_is_rejected(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
make_source_row(bank_status="pending")
|
||||
|
||||
def test_draft_declared_totals_default_to_none(self) -> None:
|
||||
draft = make_draft()
|
||||
assert draft.declared_inflow_kopecks is None
|
||||
assert draft.declared_outflow_kopecks is None
|
||||
assert draft.declared_hold_kopecks is None
|
||||
|
||||
def test_per_contract_parser_versions(self) -> None:
|
||||
assert PARSER_VERSION == "1"
|
||||
assert ALFA_DEBIT_XLSX_VERSION == "2"
|
||||
assert OZON_DEBIT_PDF_VERSION == "2"
|
||||
assert ROCKETBANK_DEBIT_PDF_VERSION == "2"
|
||||
assert AlfaDebitXlsxAdapter.parser_version == ALFA_DEBIT_XLSX_VERSION
|
||||
assert OzonDebitPdfAdapter.parser_version == OZON_DEBIT_PDF_VERSION
|
||||
assert RocketbankDebitPdfAdapter.parser_version == ROCKETBANK_DEBIT_PDF_VERSION
|
||||
|
||||
def test_registry_lists_only_verified_contracts(self) -> None:
|
||||
assert [adapter.adapter_id for adapter in ADAPTERS] == [
|
||||
"alfa_debit_xlsx",
|
||||
"ozon_debit_pdf",
|
||||
"rocketbank_debit_pdf",
|
||||
]
|
||||
|
||||
def test_credit_adapters_stay_present_but_unregistered(self) -> None:
|
||||
assert AlfaCreditXlsxAdapter.adapter_id == "alfa_credit_xlsx"
|
||||
assert OzonCreditPdfAdapter.adapter_id == "ozon_credit_pdf"
|
||||
assert all(
|
||||
adapter.adapter_id not in {"alfa_credit_xlsx", "ozon_credit_pdf"}
|
||||
for adapter in ADAPTERS
|
||||
)
|
||||
|
||||
def test_detect_adapter_fails_closed_on_unknown_xlsx(self) -> None:
|
||||
buffer = io.BytesIO()
|
||||
Workbook().save(buffer)
|
||||
raw = buffer.getvalue()
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
detect_adapter(raw, "xlsx")
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_detect_adapter_fails_closed_on_unknown_pdf(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
detect_adapter(minimal_pdf(), "pdf")
|
||||
|
||||
def test_magic_kind(self) -> None:
|
||||
assert magic_kind(b"PK\x03\x04") == "xlsx"
|
||||
assert magic_kind(b"%PDF-1.4") == "pdf"
|
||||
assert magic_kind(b"not a statement") is None
|
||||
|
||||
|
||||
class TestMoneyParsing:
|
||||
def test_plain_and_separated_amounts(self) -> None:
|
||||
assert money_to_kopecks("123") == 12300
|
||||
assert money_to_kopecks(f"1{NBSP}234,56") == 123456
|
||||
assert money_to_kopecks("1 234.56") == 123456
|
||||
assert money_to_kopecks("-1 234,56 RUR") == -123456
|
||||
assert money_to_kopecks("+ 500.00 \u20bd") == 50000
|
||||
|
||||
def test_malformed_amount_fails_closed_without_leaking_digits(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
money_to_kopecks("12a3,45")
|
||||
assert "#" in exc.value.message
|
||||
assert "12" not in exc.value.message
|
||||
|
||||
|
||||
def alfa_workbook_bytes(rows: list[list[object]]) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
workbook = Workbook()
|
||||
sheet = workbook.worksheets[0]
|
||||
for row in rows:
|
||||
sheet.append(row)
|
||||
workbook.save(buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def synthetic_alfa_rows() -> list[list[object]]:
|
||||
return [
|
||||
[None, None, None],
|
||||
["Выписка по счету"],
|
||||
["За период с 01.09.2026 по 04.09.2026"],
|
||||
["Номер счета", None, "00000000000000000001"],
|
||||
["Дата открытия счета", None, "01.01.2025"],
|
||||
["Валюта счета", None, "RUR"],
|
||||
["Тип счета", None, "Текущий счёт"],
|
||||
["Дата формирования выписки", None, "10.09.2026"],
|
||||
[
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Поступления",
|
||||
None,
|
||||
"1 000,00 RUR",
|
||||
],
|
||||
[None, None, None, None, None, None, None, None, None, None, "Расходы", None, "293,45 RUR"],
|
||||
[
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Неподтвержденные операции",
|
||||
None,
|
||||
"100,00 RUR",
|
||||
],
|
||||
[
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Текущий баланс",
|
||||
None,
|
||||
"800,00 RUR",
|
||||
],
|
||||
["Операции по счету"],
|
||||
[
|
||||
"Дата операции",
|
||||
"Дата проводки",
|
||||
None,
|
||||
"Код",
|
||||
"Категория",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Описание",
|
||||
f"Сумма в{NBSP}валюте{NBSP}счета",
|
||||
None,
|
||||
"Статус",
|
||||
],
|
||||
[
|
||||
"01.09.2026",
|
||||
"01.09.2026",
|
||||
None,
|
||||
"000000000000001",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Магазин 1 MCC: 5411",
|
||||
"-123,45",
|
||||
"Выполнен",
|
||||
],
|
||||
[
|
||||
"02.09.2026",
|
||||
"02.09.2026",
|
||||
None,
|
||||
"000000000000002",
|
||||
"Переводы",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Перевод 2 MCC5912",
|
||||
"1 000,00",
|
||||
"Выполнен",
|
||||
],
|
||||
[
|
||||
"03.09.2026",
|
||||
"HOLD",
|
||||
None,
|
||||
"000000000000003",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Покупка 3",
|
||||
"-100,00",
|
||||
],
|
||||
[
|
||||
"04.09.2026",
|
||||
"04.09.2026",
|
||||
None,
|
||||
"000000000000004",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Покупка 4 MCC 5941",
|
||||
"-50,00",
|
||||
"Выполнен",
|
||||
],
|
||||
[
|
||||
"05.09.2026",
|
||||
"05.09.2026",
|
||||
None,
|
||||
"000000000000005",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Покупка 5",
|
||||
"-20,00",
|
||||
"Выполнен",
|
||||
],
|
||||
[None],
|
||||
[
|
||||
"(подпись сотрудника АО «БАНК»)",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Уполномоченное лицо",
|
||||
],
|
||||
["Страница 1 из 1"],
|
||||
]
|
||||
|
||||
|
||||
class TestAlfaDebitContract:
|
||||
def setup_method(self) -> None:
|
||||
self.adapter = AlfaDebitXlsxAdapter()
|
||||
self.raw = alfa_workbook_bytes(synthetic_alfa_rows())
|
||||
|
||||
def test_detect(self) -> None:
|
||||
assert self.adapter.detect(self.raw) is True
|
||||
|
||||
def test_detect_rejects_workbook_without_contract_header(self) -> None:
|
||||
raw = alfa_workbook_bytes([["Выписка по счету"], ["Что-то другое", "123"]])
|
||||
assert self.adapter.detect(raw) is False
|
||||
|
||||
def test_parse_contract_fields(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert draft.adapter_id == "alfa_debit_xlsx"
|
||||
assert draft.parser_version == ALFA_DEBIT_XLSX_VERSION
|
||||
assert draft.bank_code == "alfa"
|
||||
assert draft.product == "debit"
|
||||
assert draft.currency == "RUB"
|
||||
assert draft.account_fingerprint == account_fingerprint("alfa", "00000000000000000001")
|
||||
assert draft.declared_inflow_kopecks == 100000
|
||||
# canonical posted-only: raw 293,45 minus declared hold 100,00
|
||||
assert draft.declared_outflow_kopecks == 19345
|
||||
assert draft.declared_hold_kopecks == 10000
|
||||
assert draft.closing_balance_kopecks == 80000
|
||||
assert draft.opening_balance_kopecks is None
|
||||
# exact header period fields, not min/max of transaction dates
|
||||
assert draft.period_start == date(2026, 9, 1)
|
||||
assert draft.period_end == date(2026, 9, 4)
|
||||
|
||||
def test_missing_period_field_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows[2] = ["Что-то другое"]
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
assert "period" in exc.value.message
|
||||
|
||||
def test_inconsistent_expense_control_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows[9] = [
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Расходы",
|
||||
None,
|
||||
"50,00 RUR",
|
||||
]
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
assert "expense" in exc.value.message
|
||||
|
||||
def test_parse_rows_with_hold(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert len(draft.rows) == 5
|
||||
first, second, hold, fourth, last = draft.rows
|
||||
assert (first.amount_kopecks, first.direction, first.bank_status) == (
|
||||
12345,
|
||||
"out",
|
||||
"posted",
|
||||
)
|
||||
assert first.posted_on == date(2026, 9, 1)
|
||||
assert (second.amount_kopecks, second.direction) == (100000, "in")
|
||||
assert hold.bank_status == BANK_STATUS_HOLD
|
||||
assert hold.posted_on is None
|
||||
assert hold.amount_kopecks == 10000
|
||||
assert hold.direction == "out"
|
||||
assert last.warnings == ()
|
||||
assert first.row_ref == "sheet0:row15"
|
||||
assert first.bank_category == "Покупки"
|
||||
|
||||
def test_external_ids_from_code_column(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert [row.external_id for row in draft.rows] == [
|
||||
"000000000000001",
|
||||
"000000000000002",
|
||||
"000000000000003",
|
||||
"000000000000004",
|
||||
"000000000000005",
|
||||
]
|
||||
|
||||
def test_mcc_from_description(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
first, second, hold, fourth, last = draft.rows
|
||||
assert first.mcc == 5411
|
||||
assert second.mcc == 5912
|
||||
assert hold.mcc is None
|
||||
assert fourth.mcc == 5941
|
||||
assert last.mcc is None
|
||||
assert all(row.location is None for row in draft.rows)
|
||||
|
||||
def test_footer_is_excluded(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert all(row.occurred_on is not None for row in draft.rows)
|
||||
assert draft.rows[-1].occurred_on == date(2026, 9, 5)
|
||||
|
||||
def test_parse_is_deterministic(self) -> None:
|
||||
assert self.adapter.parse(self.raw) == self.adapter.parse(self.raw)
|
||||
|
||||
def test_service_reconciliation_contract(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
summary = _reconciliation_summary(draft)
|
||||
assert summary["matched"] is True
|
||||
assert summary["posted_totals_matched"] is True
|
||||
assert summary["hold_totals_matched"] is True
|
||||
|
||||
def test_service_reconciliation_flags_mismatch(self) -> None:
|
||||
draft = make_draft(
|
||||
declared_inflow_kopecks=1000,
|
||||
declared_outflow_kopecks=1,
|
||||
declared_hold_kopecks=0,
|
||||
rows=(make_source_row(amount_kopecks=1000, direction="in"),),
|
||||
)
|
||||
summary = _reconciliation_summary(draft)
|
||||
assert summary["posted_totals_matched"] is False
|
||||
assert summary["matched"] is False
|
||||
|
||||
def test_malformed_layout_fails_closed(self) -> None:
|
||||
raw = alfa_workbook_bytes([["Дата", "Сумма"], ["01.09.2026", "-10,00"]])
|
||||
assert self.adapter.detect(raw) is False
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(raw)
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_unexpected_mid_table_row_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows.insert(16, ["unexpected layout drift"])
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_unsupported_currency_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows[5][2] = "USD"
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
|
||||
def test_row_quota_is_enforced(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(alfa_module, "get_settings_row_cap", lambda: 3)
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(self.raw)
|
||||
assert "quota" in exc.value.message
|
||||
|
||||
|
||||
def ozon_extracted() -> tuple[list[str], list[list[list[object]]]]:
|
||||
pages_text = [
|
||||
"\n".join(
|
||||
[
|
||||
"ООО «ОЗОН Банк», ИНН 1234567890, город Москва",
|
||||
"Справка о движении средств",
|
||||
"Номер лицевого счёта: № 00000000000000000001, открыт 01.01.2025",
|
||||
"Период выписки: 01.02.2026 – 28.02.2026",
|
||||
"Входящий остаток: 1000.50 \u20bd",
|
||||
]
|
||||
),
|
||||
"",
|
||||
"\n".join(
|
||||
[
|
||||
"Итого зачислений за период: 3000.00 \u20bd",
|
||||
"Итого списаний за период: 2500.75 \u20bd",
|
||||
"Исходящий остаток: 1499.75 \u20bd",
|
||||
]
|
||||
),
|
||||
]
|
||||
pages_tables = [
|
||||
[
|
||||
[
|
||||
[
|
||||
"Дата операции",
|
||||
"Документ",
|
||||
"Назначение платежа",
|
||||
"Сумма операции",
|
||||
None,
|
||||
],
|
||||
[None, None, None, "Российские рубли", "Валюта"],
|
||||
[
|
||||
"01.02.2026 10:00:00",
|
||||
"1234567890\n1",
|
||||
"Оплата товаров\nпо карте 1234",
|
||||
"- 500.00 \u20bd",
|
||||
"- 500.00 \u20bd",
|
||||
],
|
||||
]
|
||||
],
|
||||
[
|
||||
[
|
||||
[
|
||||
"02.02.2026 11:30:00",
|
||||
"1234567891",
|
||||
"Перевод\nчерез СБП",
|
||||
"+ 2 000,00 \u20bd",
|
||||
"+ 2 000,00 \u20bd",
|
||||
]
|
||||
]
|
||||
],
|
||||
[],
|
||||
]
|
||||
return pages_text, pages_tables
|
||||
|
||||
|
||||
class TestOzonDebitContract:
|
||||
def setup_method(self) -> None:
|
||||
self.adapter = OzonDebitPdfAdapter()
|
||||
self.pages_text, self.pages_tables = ozon_extracted()
|
||||
|
||||
def test_core_contract_fields(self) -> None:
|
||||
draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables)
|
||||
assert draft.adapter_id == "ozon_debit_pdf"
|
||||
assert draft.bank_code == "ozon"
|
||||
assert draft.currency == "RUB"
|
||||
assert draft.account_fingerprint == account_fingerprint("ozon", "00000000000000000001")
|
||||
assert draft.period_start == date(2026, 2, 1)
|
||||
assert draft.period_end == date(2026, 2, 28)
|
||||
assert draft.opening_balance_kopecks == 100050
|
||||
assert draft.closing_balance_kopecks == 149975
|
||||
assert draft.declared_inflow_kopecks == 300000
|
||||
assert draft.declared_outflow_kopecks == 250075
|
||||
assert draft.declared_hold_kopecks is None
|
||||
|
||||
def test_multiline_cells_stay_one_transaction(self) -> None:
|
||||
draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables)
|
||||
assert len(draft.rows) == 2
|
||||
first, second = draft.rows
|
||||
assert first.external_id == "12345678901"
|
||||
assert first.description == "Оплата товаров по карте 1234"
|
||||
assert (first.amount_kopecks, first.direction) == (50000, "out")
|
||||
assert first.occurred_on == date(2026, 2, 1)
|
||||
assert (second.amount_kopecks, second.direction) == (200000, "in")
|
||||
assert second.external_id == "1234567891"
|
||||
assert first.warnings == ()
|
||||
|
||||
def test_missing_header_marker_fails_closed(self) -> None:
|
||||
pages_text, pages_tables = ozon_extracted()
|
||||
pages_text[0] = "Какой-то другой документ"
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def test_missing_totals_fail_closed(self) -> None:
|
||||
pages_text, pages_tables = ozon_extracted()
|
||||
pages_text[2] = ""
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def test_no_rows_fails_closed(self) -> None:
|
||||
pages_text, _ = ozon_extracted()
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, [[]])
|
||||
|
||||
def test_unexpected_row_shape_fails_closed(self) -> None:
|
||||
pages_text, pages_tables = ozon_extracted()
|
||||
pages_tables[0][0].append([None, None, "текст без суммы"])
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None:
|
||||
raw = minimal_pdf()
|
||||
assert self.adapter.detect(raw) is False
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(raw)
|
||||
assert "text layer" in exc.value.message
|
||||
|
||||
def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import app.imports.readers as readers_module
|
||||
|
||||
monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings())
|
||||
with pytest.raises(AppError) as exc:
|
||||
self.adapter.parse(minimal_pdf(page_count=2))
|
||||
assert exc.value.code == "IMPORT_QUOTA_EXCEEDED"
|
||||
|
||||
|
||||
def same_page_rocket_lines() -> list[tuple[int, str]]:
|
||||
return [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "С 01.07.2026 ПО 10.09.2026"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ"),
|
||||
(1, "0,00 RUR 0,00 RUR"),
|
||||
(1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"),
|
||||
(1, "0,00 RUR 0,00 RUR"),
|
||||
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
|
||||
(1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
|
||||
(1, "10:00 00000000001 -10,00 RUR"),
|
||||
(1, "11:00 00000000002 -20,00 RUR"),
|
||||
(1, "12:00 00000000003 -30,00 RUR"),
|
||||
(1, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"),
|
||||
(1, "ВСЕГО РАСХОДОВ: 60,00 RUR"),
|
||||
]
|
||||
|
||||
|
||||
def synthetic_rocket_lines() -> list[tuple[int, str]]:
|
||||
return [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"),
|
||||
(1, "С 01.07.2026 ПО 10.09.2026"),
|
||||
(1, "ИВАНОВ ИВАН ИВАНОВИЧ"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ ДАТА ПОСЛЕДНЕГО ДВИЖЕНИЯ"),
|
||||
(1, "0,00 RUR 1 000,00 RUR 01.07.2026"),
|
||||
(1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"),
|
||||
(1, "500,00 RUR 500,00 RUR"),
|
||||
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
|
||||
(1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
|
||||
(1, "ЗАЧИСЛЕНИЕ ПЕРЕВОДА ДЕНЕЖНЫХ СРЕДСТВ/СБП/"),
|
||||
(1, "12:00 00000000001 +1 000,00 RUR"),
|
||||
(1, "СЧЕТ КОРРЕСПОНДЕНТА БАНК ВТБ (ПАО)."),
|
||||
(1, "ВСЕГО ПОСТУПЛЕНИЙ: 1 000,00 RUR"),
|
||||
(1, "ВСЕГО РАСХОДОВ: 0,00 RUR"),
|
||||
(1, "СТРАНИЦА 1/2"),
|
||||
(2, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(2, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"),
|
||||
(2, "С 01.07.2026 ПО 10.09.2026"),
|
||||
(2, "02 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 1 000,00 RUR"),
|
||||
(2, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
|
||||
(2, "ПЛАТЕЖ АВТОРИЗАЦИЯ №123,01.07.2026 10:00,0000********0000,"),
|
||||
(2, "13:30 00000000002 555RUR,EKATERINBURG,MCC 5411,SHOP\\RU\\CITY\\SHOP\\ -250,50 RUR"),
|
||||
(2, "0000\\ ЗДАНИЕ."),
|
||||
(2, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"),
|
||||
(2, "ВСЕГО РАСХОДОВ: 250,50 RUR"),
|
||||
(2, "СТРАНИЦА 2/2"),
|
||||
]
|
||||
|
||||
|
||||
class TestRocketbankDebitContract:
|
||||
def setup_method(self) -> None:
|
||||
self.adapter = RocketbankDebitPdfAdapter()
|
||||
|
||||
def test_day_and_page_state_machine(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
assert len(draft.rows) == 2
|
||||
first, second = draft.rows
|
||||
assert first.occurred_on == date(2026, 7, 1)
|
||||
assert second.occurred_on == date(2026, 7, 2)
|
||||
|
||||
def test_row_refs_are_unique_within_a_page(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
first, second = draft.rows
|
||||
assert first.row_ref == "page1:record1"
|
||||
assert second.row_ref == "page2:record1"
|
||||
same_page = self.adapter._draft_from_lines(same_page_rocket_lines())
|
||||
refs = [row.row_ref for row in same_page.rows]
|
||||
assert refs == ["page1:record1", "page1:record2", "page1:record3"]
|
||||
assert len(set(refs)) == len(refs)
|
||||
|
||||
def test_statement_controls(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
assert draft.adapter_id == "rocketbank_debit_pdf"
|
||||
assert draft.bank_code == "rocketbank"
|
||||
assert draft.currency == "RUB"
|
||||
assert draft.account_fingerprint == account_fingerprint(
|
||||
"rocketbank", "00000000000000000001"
|
||||
)
|
||||
assert draft.period_start == date(2026, 7, 1)
|
||||
assert draft.period_end == date(2026, 9, 10)
|
||||
assert draft.opening_balance_kopecks == 0
|
||||
assert draft.closing_balance_kopecks == 50000
|
||||
assert draft.declared_inflow_kopecks == 100000
|
||||
assert draft.declared_outflow_kopecks == 50000
|
||||
assert draft.declared_hold_kopecks is None
|
||||
|
||||
def test_record_folding_and_enrichment(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
first, second = draft.rows
|
||||
assert first.external_id == "00000000001"
|
||||
assert (first.amount_kopecks, first.direction) == (100000, "in")
|
||||
assert "ЗАЧИСЛЕНИЕ ПЕРЕВОДА" in (first.description or "")
|
||||
assert "СЧЕТ КОРРЕСПОНДЕНТА" in (first.description or "")
|
||||
assert first.mcc is None
|
||||
assert second.external_id == "00000000002"
|
||||
assert (second.amount_kopecks, second.direction) == (25050, "out")
|
||||
assert second.mcc == 5411
|
||||
assert second.location == "EKATERINBURG"
|
||||
assert (second.description or "").startswith("ПЛАТЕЖ АВТОРИЗАЦИЯ")
|
||||
assert "555RUR" in (second.description or "")
|
||||
assert "ЗДАНИЕ." in (second.description or "")
|
||||
assert all(row.warnings == () for row in draft.rows)
|
||||
|
||||
def test_repeated_headers_and_daily_totals_are_ignored(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
joined = " ".join(row.description or "" for row in draft.rows)
|
||||
assert "ВСЕГО" not in joined
|
||||
assert "ВЫПИСКА ПО СЧЕТУ" not in joined
|
||||
assert "СТРАНИЦА" not in joined
|
||||
|
||||
def test_parse_is_deterministic(self) -> None:
|
||||
assert self.adapter._draft_from_lines(
|
||||
synthetic_rocket_lines()
|
||||
) == self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
|
||||
def test_record_before_day_section_fails_closed(self) -> None:
|
||||
lines = [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "12:00 00000000001 +1 000,00 RUR"),
|
||||
]
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_lines(lines)
|
||||
|
||||
def test_missing_header_block_fails_closed(self) -> None:
|
||||
lines = [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
|
||||
(1, "12:00 00000000001 +1 000,00 RUR"),
|
||||
]
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_lines(lines)
|
||||
|
||||
def test_unrecognized_content_fails_closed(self) -> None:
|
||||
lines = synthetic_rocket_lines()[:11] + [(1, "НЕИЗВЕСТНАЯ СТРОКА БЕЗ СТРУКТУРЫ")]
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter._draft_from_lines(lines)
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_fold_line_cap_fails_closed(self) -> None:
|
||||
junk = [(1, f"НЕИЗВЕСТНАЯ СТРОКА {index}") for index in range(7)]
|
||||
lines = synthetic_rocket_lines()[:11] + junk
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_lines(lines)
|
||||
|
||||
def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None:
|
||||
raw = minimal_pdf()
|
||||
assert self.adapter.detect(raw) is False
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(raw)
|
||||
assert "text layer" in exc.value.message
|
||||
|
||||
def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import app.imports.readers as readers_module
|
||||
|
||||
monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings())
|
||||
with pytest.raises(AppError) as exc:
|
||||
self.adapter.parse(minimal_pdf(page_count=2))
|
||||
assert exc.value.code == "IMPORT_QUOTA_EXCEEDED"
|
||||
|
||||
|
||||
class _QuotaSettings:
|
||||
max_pdf_pages = 1
|
||||
|
||||
|
||||
def minimal_pdf(page_count: int = 1, page_text: str = "") -> bytes:
|
||||
"""Build a small valid single-font PDF with empty or ASCII-only pages."""
|
||||
objects: list[tuple[int, bytes]] = []
|
||||
page_ids: list[int] = []
|
||||
next_id = 3
|
||||
for _ in range(page_count):
|
||||
page_id = next_id
|
||||
content_id = next_id + 1
|
||||
next_id += 2
|
||||
page_ids.append(page_id)
|
||||
content = (
|
||||
f"BT /F1 12 Tf 72 720 Td ({page_text}) Tj ET".encode("latin-1", "replace")
|
||||
if page_text
|
||||
else b""
|
||||
)
|
||||
objects.append(
|
||||
(
|
||||
content_id,
|
||||
b"<< /Length %d >>\nstream\n%s\nendstream" % (len(content), content),
|
||||
)
|
||||
)
|
||||
objects.append(
|
||||
(
|
||||
page_id,
|
||||
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents %d 0 R >>"
|
||||
% content_id,
|
||||
)
|
||||
)
|
||||
font_id = next_id
|
||||
objects.append((font_id, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"))
|
||||
objects.insert(
|
||||
0,
|
||||
(
|
||||
2,
|
||||
b"<< /Type /Pages /Kids ["
|
||||
+ b" ".join(b"%d 0 R" % page_id for page_id in page_ids)
|
||||
+ b"] /Count %d >>" % page_count,
|
||||
),
|
||||
)
|
||||
objects.insert(0, (1, b"<< /Type /Catalog /Pages 2 0 R >>"))
|
||||
out = bytearray(b"%PDF-1.4\n")
|
||||
offsets: dict[int, int] = {}
|
||||
for number, body in objects:
|
||||
offsets[number] = len(out)
|
||||
out += b"%d 0 obj\n" % number + body + b"\nendobj\n"
|
||||
xref_position = len(out)
|
||||
size = max(offsets) + 1
|
||||
out += b"xref\n0 %d\n" % size
|
||||
out += b"0000000000 65535 f \n"
|
||||
for number in range(1, size):
|
||||
out += b"%010d 00000 n \n" % offsets[number]
|
||||
out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
|
||||
size,
|
||||
xref_position,
|
||||
)
|
||||
return bytes(out)
|
||||
@@ -0,0 +1,945 @@
|
||||
import uuid
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.budgeting.service import close_period, compute_period_state, get_or_create_period
|
||||
from app.errors import AppError
|
||||
from app.identity.models import User
|
||||
from app.imports import service
|
||||
from app.imports.adapters.base import record_fingerprint
|
||||
from app.imports.models import (
|
||||
HoldResolution,
|
||||
ParseRun,
|
||||
SourceRecord,
|
||||
SourceRecordLink,
|
||||
StatementDocument,
|
||||
)
|
||||
from app.ledger.models import Operation
|
||||
from app.ledger.service import create_account
|
||||
from app.spaces.models import Membership
|
||||
from app.spaces.service import create_space
|
||||
|
||||
MAY_10 = date(2026, 5, 10)
|
||||
|
||||
|
||||
def seed(db: Session, *, name: str = "Holds"):
|
||||
user = User(email=f"{uuid.uuid4().hex}@example.com", password_hash="unused")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
space = create_space(db, user=user, name=name)
|
||||
account = create_account(
|
||||
db,
|
||||
space_id=space.id,
|
||||
user=user,
|
||||
name="Debit",
|
||||
kind="debit",
|
||||
opened_on=date(2026, 1, 1),
|
||||
)
|
||||
return user, space, account
|
||||
|
||||
|
||||
def make_document(
|
||||
db: Session,
|
||||
space,
|
||||
account,
|
||||
*,
|
||||
user: User | None = None,
|
||||
status: str = "ready",
|
||||
adapter_id: str = "alfa_debit_xlsx",
|
||||
) -> tuple[StatementDocument, ParseRun]:
|
||||
document = StatementDocument(
|
||||
space_id=space.id,
|
||||
account_id=account.id,
|
||||
original_filename=f"{uuid.uuid4().hex}.xlsx",
|
||||
size_bytes=100,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status=status,
|
||||
reconciliation={"full": True, "matched": True},
|
||||
uploaded_by_user_id=user.id if user else None,
|
||||
)
|
||||
db.add(document)
|
||||
db.flush()
|
||||
run = ParseRun(
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
adapter_id=adapter_id,
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
document.active_parse_run_id = run.id
|
||||
document.adapter_id = adapter_id
|
||||
return document, run
|
||||
|
||||
|
||||
def add_record(
|
||||
db: Session,
|
||||
document: StatementDocument,
|
||||
run: ParseRun,
|
||||
*,
|
||||
direction: str = "out",
|
||||
amount: int = 12_345,
|
||||
occurred_on: date = MAY_10,
|
||||
bank_status: str = "posted",
|
||||
external_id: str | None = None,
|
||||
description: str | None = None,
|
||||
) -> SourceRecord:
|
||||
record = SourceRecord(
|
||||
space_id=document.space_id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
external_id=external_id,
|
||||
occurred_on=occurred_on,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
bank_status=bank_status,
|
||||
description=description,
|
||||
row_ref=f"sheet0:{uuid.uuid4().hex[:8]}",
|
||||
record_fp=record_fingerprint(occurred_on, amount, description),
|
||||
)
|
||||
db.add(record)
|
||||
db.flush()
|
||||
return record
|
||||
|
||||
|
||||
def hold_document(db: Session, *, occurred_on: date = MAY_10, external_id: str | None = None):
|
||||
"""A confirmed import whose active run contains exactly one unresolved hold."""
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
document,
|
||||
run,
|
||||
bank_status="hold",
|
||||
occurred_on=occurred_on,
|
||||
external_id=external_id,
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
db.flush()
|
||||
return user, space, account, document, hold
|
||||
|
||||
|
||||
def test_bank_status_defaults_to_posted_and_is_constrained(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
record = SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
amount_kopecks=1_000,
|
||||
direction="out",
|
||||
row_ref="sheet0:row1",
|
||||
record_fp="fp",
|
||||
)
|
||||
db.add(record)
|
||||
db.flush()
|
||||
assert record.bank_status == "posted"
|
||||
bad = SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
amount_kopecks=1_000,
|
||||
direction="out",
|
||||
bank_status="pending",
|
||||
row_ref="sheet0:row2",
|
||||
record_fp="fp2",
|
||||
)
|
||||
db.add(bad)
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_hold_row_creates_no_operation_on_confirm(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
add_record(
|
||||
db,
|
||||
document,
|
||||
run,
|
||||
bank_status="posted",
|
||||
occurred_on=date(2026, 5, 11),
|
||||
amount=7_000,
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
db.flush()
|
||||
operations = db.query(Operation).filter_by(space_id=space.id).all()
|
||||
assert len(operations) == 1
|
||||
assert operations[0].amount_kopecks == 7_000
|
||||
assert db.query(SourceRecordLink).filter_by(source_record_id=hold.id).count() == 0
|
||||
assert document.status == "confirmed"
|
||||
detail = service.document_detail(db, space_id=space.id, document_id=document.id)
|
||||
hold_row = next(item for item in detail["records"] if item["id"] == hold.id)
|
||||
assert hold_row["bank_status"] == "hold"
|
||||
assert hold_row["hold_resolution"] is None
|
||||
assert hold_row["critical"] is False
|
||||
|
||||
|
||||
def test_unresolved_hold_blocks_only_its_occurred_month(db: Session) -> None:
|
||||
user, space, _account, _document, _hold = hold_document(db)
|
||||
april = get_or_create_period(db, space_id=space.id, year=2026, month=4)
|
||||
close_period(db, space_id=space.id, user=user, period=april)
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
with pytest.raises(AppError) as exc:
|
||||
close_period(db, space_id=space.id, user=user, period=may)
|
||||
assert exc.value.code == "BUDGET_CLOSE_BLOCKED"
|
||||
june = get_or_create_period(db, space_id=space.id, year=2026, month=6)
|
||||
june_state = compute_period_state(db, space_id=space.id, period=june)
|
||||
assert not any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in june_state["close_blockers"]
|
||||
)
|
||||
|
||||
|
||||
def test_discard_as_cancelled_requires_reason_and_unblocks_month(db: Session) -> None:
|
||||
user, space, _account, document, hold = hold_document(db)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason=" ",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED"
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="Cancelled by the bank",
|
||||
)
|
||||
db.flush()
|
||||
assert resolution.resolution == "discarded_as_cancelled"
|
||||
assert resolution.operation_id is None
|
||||
assert resolution.resolved_by_user_id == user.id
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
detail = service.document_detail(db, space_id=space.id, document_id=document.id)
|
||||
hold_row = next(item for item in detail["records"] if item["id"] == hold.id)
|
||||
assert hold_row["hold_resolution"]["reason"] == "Cancelled by the bank"
|
||||
april = get_or_create_period(db, space_id=space.id, year=2026, month=4)
|
||||
close_period(db, space_id=space.id, user=user, period=april)
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
close_period(db, space_id=space.id, user=user, period=may)
|
||||
assert may.status == "closed"
|
||||
|
||||
|
||||
def test_confirm_as_posted_requires_reason(db: Session) -> None:
|
||||
user, space, _account, document, hold = hold_document(db)
|
||||
for reason in (None, ""):
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason=reason,
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED"
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
|
||||
def test_hold_cannot_be_resolved_before_document_confirmation(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Not confirmed yet",
|
||||
)
|
||||
|
||||
assert exc.value.code == "IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED"
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
|
||||
def test_manual_confirm_rejects_bad_action_and_non_hold_row(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
posted = add_record(
|
||||
db, document, run, bank_status="posted", occurred_on=date(2026, 5, 11), amount=900
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="delete_forever",
|
||||
reason="why not",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_ACTION_INVALID"
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=posted.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="posted row",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_INVALID"
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=uuid.uuid4(),
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="missing row",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_NOT_FOUND"
|
||||
|
||||
|
||||
def test_manual_confirm_creates_one_operation_and_late_import_dedupes(db: Session) -> None:
|
||||
user, space, account, document, hold = hold_document(db, external_id="H-1")
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Verified against the paper receipt",
|
||||
account_id=account.id,
|
||||
)
|
||||
db.flush()
|
||||
operations = db.query(Operation).filter_by(space_id=space.id).all()
|
||||
assert len(operations) == 1
|
||||
operation = operations[0]
|
||||
assert operation.type == "expense"
|
||||
assert operation.amount_kopecks == 12_345
|
||||
assert operation.occurred_on == MAY_10
|
||||
assert resolution.operation_id == operation.id
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one_or_none()
|
||||
assert link is not None and link.operation_id == operation.id
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="second try",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_ALREADY_RESOLVED"
|
||||
|
||||
later, later_run = make_document(db, space, account, user=user)
|
||||
late_posted = add_record(
|
||||
db,
|
||||
later,
|
||||
later_run,
|
||||
bank_status="posted",
|
||||
external_id="H-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=later.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 1
|
||||
late_link = db.query(SourceRecordLink).filter_by(source_record_id=late_posted.id).one_or_none()
|
||||
assert late_link is not None
|
||||
assert late_link.operation_id == operation.id
|
||||
assert late_link.kind == "duplicate"
|
||||
|
||||
|
||||
def test_manual_confirm_of_income_hold_creates_income(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
document,
|
||||
run,
|
||||
direction="in",
|
||||
amount=45_000,
|
||||
bank_status="hold",
|
||||
description="Refund hold",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Money arrived",
|
||||
)
|
||||
db.flush()
|
||||
operation = db.query(Operation).filter_by(space_id=space.id).one()
|
||||
assert operation.type == "income"
|
||||
assert operation.amount_kopecks == 45_000
|
||||
|
||||
|
||||
def test_manual_confirm_honors_closed_period(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
february = get_or_create_period(db, space_id=space.id, year=2026, month=2)
|
||||
close_period(db, space_id=space.id, user=user, period=february)
|
||||
march = get_or_create_period(db, space_id=space.id, year=2026, month=3)
|
||||
close_period(db, space_id=space.id, user=user, period=march)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold", occurred_on=date(2026, 3, 15))
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Book it",
|
||||
)
|
||||
assert exc.value.code == "BUDGET_PERIOD_CLOSED"
|
||||
db.flush()
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="Never posted",
|
||||
)
|
||||
assert resolution.operation_id is None
|
||||
|
||||
|
||||
def test_unique_posted_record_resolves_hold_on_confirmation(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="ALFA-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="ALFA-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
resolution = db.query(HoldResolution).one()
|
||||
assert resolution.hold_source_record_id == hold.id
|
||||
assert resolution.resolving_source_record_id == posted.id
|
||||
assert resolution.resolution == "matched_posted"
|
||||
assert resolution.resolved_by_user_id is None
|
||||
operations = db.query(Operation).filter_by(space_id=space.id).all()
|
||||
assert len(operations) == 1
|
||||
assert operations[0].amount_kopecks == 12_345
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert not any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]
|
||||
)
|
||||
|
||||
|
||||
def test_external_id_match_survives_description_drift(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="X-9",
|
||||
description="HOLD COFFEE",
|
||||
occurred_on=MAY_10,
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="X-9",
|
||||
description="Coffee llc prague",
|
||||
occurred_on=date(2026, 5, 12),
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
resolution = db.query(HoldResolution).one()
|
||||
assert resolution.hold_source_record_id == hold.id
|
||||
assert resolution.resolving_source_record_id == posted.id
|
||||
|
||||
|
||||
def test_fingerprint_match_without_external_id_resolves(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
description="Coffee shop",
|
||||
)
|
||||
matches = service._match_hold_resolutions(db, document=second, run=second_run)
|
||||
assert matches == {hold.id: posted.id}
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 1
|
||||
|
||||
|
||||
def test_ambiguous_overlap_remains_unresolved(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold_a = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="A-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
hold_b = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="B-2",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
assert hold_a.id != hold_b.id
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
|
||||
|
||||
def test_matcher_rejects_two_posted_candidates_for_one_hold(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
add_record(db, first, first_run, bank_status="hold", description="Coffee shop")
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
add_record(db, second, second_run, bank_status="posted", description="Coffee shop")
|
||||
add_record(db, second, second_run, bank_status="posted", description="Coffee shop")
|
||||
matches = service._match_hold_resolutions(db, document=second, run=second_run)
|
||||
assert matches == {}
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
|
||||
def test_discarded_posted_record_does_not_resolve_hold(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="DISCARDED-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="DISCARDED-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
posted.review_decision = "discard"
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
|
||||
assert db.query(HoldResolution).filter_by(hold_source_record_id=hold.id).count() == 0
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
|
||||
|
||||
def test_hold_matching_is_tenant_scoped(db: Session) -> None:
|
||||
user1, space1, account1 = seed(db, name="Space One")
|
||||
user2, space2, account2 = seed(db, name="Space Two")
|
||||
first, first_run = make_document(db, space1, account1, user=user1)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="T-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space1.id, document_id=first.id, user=user1)
|
||||
second, second_run = make_document(db, space2, account2, user=user2)
|
||||
add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="T-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space2.id, document_id=second.id, user=user2)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
may_two = get_or_create_period(db, space_id=space2.id, year=2026, month=5)
|
||||
state_two = compute_period_state(db, space_id=space2.id, period=may_two)
|
||||
assert not any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_two["close_blockers"]
|
||||
)
|
||||
may_one = get_or_create_period(db, space_id=space1.id, year=2026, month=5)
|
||||
state_one = compute_period_state(db, space_id=space1.id, period=may_one)
|
||||
assert any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_one["close_blockers"]
|
||||
)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space2.id,
|
||||
document_id=first.id,
|
||||
source_record_id=hold.id,
|
||||
user=user2,
|
||||
action="discard_as_cancelled",
|
||||
reason="cross tenant",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_NOT_FOUND"
|
||||
|
||||
|
||||
def test_revert_clears_automatic_match_and_reblocks_month(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="R-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="R-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 1
|
||||
service.revert_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
assert hold.bank_status == "hold"
|
||||
|
||||
|
||||
def test_revert_preserves_manual_hold_operation_and_provenance(db: Session) -> None:
|
||||
user, space, account, document, hold = hold_document(db)
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Verified in the bank app",
|
||||
account_id=account.id,
|
||||
)
|
||||
db.flush()
|
||||
operation_id = resolution.operation_id
|
||||
|
||||
service.revert_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
db.flush()
|
||||
|
||||
assert document.status == "ready"
|
||||
assert operation_id is not None
|
||||
assert db.get(Operation, operation_id) is not None
|
||||
assert db.get(HoldResolution, resolution.id) is not None
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one()
|
||||
assert link.kind == "hold_confirmed"
|
||||
assert link.operation_id == operation_id
|
||||
|
||||
|
||||
def test_hold_resolution_rejects_unknown_kind(db: Session) -> None:
|
||||
user, space, _account, _document, hold = hold_document(db)
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space.id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="resolved_somehow",
|
||||
)
|
||||
)
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_manual_resolution_requires_reason_at_database_level(db: Session) -> None:
|
||||
_user, space, _account, _document, hold = hold_document(db)
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space.id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="confirmed_as_posted",
|
||||
reason=None,
|
||||
)
|
||||
)
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
_user, space, _account, _document, hold = hold_document(db)
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space.id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="matched_posted",
|
||||
reason=None,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _draft(rows, *, opening=1_000, closing=1_200, **declared):
|
||||
return SimpleNamespace(
|
||||
currency="RUB",
|
||||
opening_balance_kopecks=opening,
|
||||
closing_balance_kopecks=closing,
|
||||
declared_inflow_kopecks=declared.get("inflow"),
|
||||
declared_outflow_kopecks=declared.get("outflow"),
|
||||
declared_hold_kopecks=declared.get("hold"),
|
||||
rows=tuple(rows),
|
||||
)
|
||||
|
||||
|
||||
def _row(amount, direction, bank_status="posted"):
|
||||
return SimpleNamespace(
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
bank_status=bank_status,
|
||||
)
|
||||
|
||||
|
||||
def test_reconciliation_separates_posted_and_hold_totals() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft(
|
||||
[
|
||||
_row(50_000, "in"),
|
||||
_row(30_000, "out"),
|
||||
_row(20_000, "out", bank_status="hold"),
|
||||
],
|
||||
opening=100_000,
|
||||
closing=120_000,
|
||||
inflow=50_000,
|
||||
outflow=30_000,
|
||||
hold=20_000,
|
||||
)
|
||||
)
|
||||
assert summary["parsed_posted_inflow_kopecks"] == 50_000
|
||||
assert summary["parsed_posted_outflow_kopecks"] == 30_000
|
||||
assert summary["parsed_hold_total_kopecks"] == 20_000
|
||||
assert summary["posted_totals_matched"] is True
|
||||
assert summary["hold_totals_matched"] is True
|
||||
assert summary["balance_matched"] is True
|
||||
assert summary["matched"] is True
|
||||
assert summary["computed_closing_kopecks"] == 120_000
|
||||
|
||||
|
||||
def test_reconciliation_declared_outflow_mismatch_blocks() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft(
|
||||
[_row(30_000, "out")],
|
||||
opening=100_000,
|
||||
closing=70_000,
|
||||
inflow=None,
|
||||
outflow=40_000,
|
||||
)
|
||||
)
|
||||
assert summary["posted_totals_matched"] is False
|
||||
assert summary["hold_totals_matched"] is None
|
||||
assert summary["matched"] is False
|
||||
|
||||
|
||||
def test_reconciliation_hold_mismatch_is_review_blocking() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft([_row(25_000, "out", bank_status="hold")], opening=None, closing=None, hold=10_000)
|
||||
)
|
||||
assert summary["full"] is False
|
||||
assert summary["hold_totals_matched"] is False
|
||||
assert summary["matched"] is False
|
||||
assert summary["balance_matched"] is None
|
||||
|
||||
|
||||
def test_reconciliation_balance_equation_ignores_holds() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft(
|
||||
[
|
||||
_row(50_000, "in"),
|
||||
_row(30_000, "out"),
|
||||
_row(20_000, "out", bank_status="hold"),
|
||||
],
|
||||
opening=100_000,
|
||||
closing=140_000,
|
||||
)
|
||||
)
|
||||
assert summary["balance_matched"] is False
|
||||
assert summary["matched"] is False
|
||||
assert summary["computed_closing_kopecks"] == 120_000
|
||||
|
||||
|
||||
def test_reconciliation_without_declared_totals_keeps_legacy_semantics() -> None:
|
||||
draft = SimpleNamespace(
|
||||
currency="RUB",
|
||||
opening_balance_kopecks=100,
|
||||
closing_balance_kopecks=300,
|
||||
rows=(_row(200, "in"),),
|
||||
)
|
||||
summary = service._reconciliation_summary(draft)
|
||||
assert summary["full"] is True
|
||||
assert summary["posted_totals_matched"] is None
|
||||
assert summary["hold_totals_matched"] is None
|
||||
assert summary["balance_matched"] is True
|
||||
assert summary["matched"] is True
|
||||
|
||||
|
||||
def test_source_row_without_bank_status_defaults_posted() -> None:
|
||||
from app.imports.adapters.base import SourceRow
|
||||
|
||||
source = service._source_from_row(
|
||||
space_id=uuid.uuid4(),
|
||||
document_id=uuid.uuid4(),
|
||||
run_id=uuid.uuid4(),
|
||||
row=SourceRow(
|
||||
external_id=None,
|
||||
occurred_on=date(2026, 5, 10),
|
||||
posted_on=None,
|
||||
amount_kopecks=1_000,
|
||||
direction="out",
|
||||
description=None,
|
||||
mcc=None,
|
||||
bank_category=None,
|
||||
location=None,
|
||||
row_ref="row1",
|
||||
),
|
||||
)
|
||||
assert source.bank_status == "posted"
|
||||
assert source.critical is False
|
||||
|
||||
|
||||
def test_resolve_endpoint_requires_csrf_and_resolves_hold(client: TestClient, db: Session) -> None:
|
||||
invite = client.post(
|
||||
"/api/v1/admin/invitations",
|
||||
headers={"X-Admin-Token": "test-admin-token"},
|
||||
json={"email": "holds-user@example.com"},
|
||||
)
|
||||
assert invite.status_code == 201, invite.text
|
||||
accepted = client.post(
|
||||
"/api/v1/auth/invitations/accept",
|
||||
json={
|
||||
"token": invite.json()["token"],
|
||||
"password": "a-long-password-for-tests",
|
||||
"fullName": "Holds User",
|
||||
"locale": "en",
|
||||
},
|
||||
)
|
||||
assert accepted.status_code == 201, accepted.text
|
||||
csrf = client.cookies["csrf_token"]
|
||||
db.commit()
|
||||
|
||||
user = db.query(User).filter_by(email="holds-user@example.com").one()
|
||||
space_id = db.query(Membership).filter_by(user_id=user.id).one().space_id
|
||||
account = create_account(
|
||||
db,
|
||||
space_id=space_id,
|
||||
user=user,
|
||||
name="Debit",
|
||||
kind="debit",
|
||||
opened_on=date(2026, 1, 1),
|
||||
)
|
||||
space = SimpleNamespace(id=space_id)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
service.confirm_document(db, space_id=space_id, document_id=document.id, user=user)
|
||||
db.commit()
|
||||
|
||||
rejected = client.post(
|
||||
f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve",
|
||||
json={"action": "confirm_as_posted", "reason": "no csrf"},
|
||||
)
|
||||
assert rejected.status_code == 403
|
||||
|
||||
resolved = client.post(
|
||||
f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve",
|
||||
headers={"X-CSRF-Token": csrf},
|
||||
json={"action": "confirm_as_posted", "reason": "Checked in the bank app"},
|
||||
)
|
||||
assert resolved.status_code == 200, resolved.text
|
||||
rows = resolved.json()["rows"]
|
||||
row = next(item for item in rows if item["id"] == str(hold.id))
|
||||
assert row["bankStatus"] == "hold"
|
||||
assert row["holdResolution"]["resolution"] == "confirmed_as_posted"
|
||||
assert row["holdResolution"]["reason"] == "Checked in the bank app"
|
||||
assert row["holdResolution"]["operationId"] is not None
|
||||
operations = db.query(Operation).filter_by(space_id=space_id).all()
|
||||
assert len(operations) == 1
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import set_space_context
|
||||
from app.imports.blob import get_blob_store
|
||||
from app.imports.models import HoldResolution, ParseRun, SourceRecord, StatementDocument
|
||||
from app.ledger.models import Account
|
||||
from app.spaces.models import FinancialSpace
|
||||
|
||||
@@ -28,11 +29,55 @@ def test_postgres_rls_requires_matching_transaction_context() -> None:
|
||||
db.rollback()
|
||||
|
||||
set_space_context(db, space_id)
|
||||
db.add(Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today()))
|
||||
account = Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today())
|
||||
db.add(account)
|
||||
db.flush()
|
||||
document = StatementDocument(
|
||||
space_id=space_id,
|
||||
account_id=account.id,
|
||||
original_filename="synthetic.xlsx",
|
||||
size_bytes=1,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status="confirmed",
|
||||
)
|
||||
db.add(document)
|
||||
db.flush()
|
||||
run = ParseRun(
|
||||
space_id=space_id,
|
||||
document_id=document.id,
|
||||
adapter_id="alfa_debit_xlsx",
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
document.active_parse_run_id = run.id
|
||||
hold = SourceRecord(
|
||||
space_id=space_id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
occurred_on=date.today(),
|
||||
amount_kopecks=100,
|
||||
direction="out",
|
||||
bank_status="hold",
|
||||
row_ref="sheet0:row1",
|
||||
record_fp=uuid.uuid4().hex,
|
||||
)
|
||||
db.add(hold)
|
||||
db.flush()
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space_id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="discarded_as_cancelled",
|
||||
reason="synthetic test resolution",
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
set_space_context(db, uuid.uuid4())
|
||||
assert db.query(Account).count() == 0
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
db.rollback()
|
||||
set_space_context(db, space_id)
|
||||
persisted_space = db.get(FinancialSpace, space_id)
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Private real-statement suite (marker: private_fixtures).
|
||||
|
||||
Runs only when invoked explicitly:
|
||||
|
||||
PRIVATE_FIXTURES_DIR=/secure/path uv run pytest -m private_fixtures
|
||||
|
||||
When PRIVATE_FIXTURES_DIR is unset, the ignored repository-local ``examples/``
|
||||
directory is used if it contains a ``manifest.json``. Normal CI never collects
|
||||
this module (the default pytest addopts deselect the marker).
|
||||
|
||||
Privacy contract (docs/parsers/private-fixture-gate.md):
|
||||
|
||||
- fixture contents, parsed rows, descriptions, identifiers, and manifest
|
||||
financial values never appear in assertion output, messages, or logs;
|
||||
- failures report only the filename, adapter id, and the name of the
|
||||
mismatched aggregate;
|
||||
- every file is parsed twice and the results must be identical.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.imports.adapters import detect_adapter, magic_kind
|
||||
from app.imports.service import _reconciliation_summary
|
||||
|
||||
pytestmark = pytest.mark.private_fixtures
|
||||
|
||||
REPO_EXAMPLES = Path(__file__).resolve().parents[2] / "examples"
|
||||
|
||||
|
||||
def _load_entries() -> tuple[Path | None, list[dict], str | None]:
|
||||
"""Resolve fixture entries.
|
||||
|
||||
Returns (directory, entries, explicit_error). An explicit error is only
|
||||
produced when PRIVATE_FIXTURES_DIR was set on purpose: then a missing
|
||||
directory, missing/malformed manifest, or empty manifest must fail loudly
|
||||
(redacted message, no values). Without the variable, an absent or broken
|
||||
repo-local examples directory is a silent skip.
|
||||
"""
|
||||
env = os.environ.get("PRIVATE_FIXTURES_DIR")
|
||||
if env:
|
||||
directory = Path(env)
|
||||
if not directory.is_dir():
|
||||
return None, [], "PRIVATE_FIXTURES_DIR is set but the directory does not exist"
|
||||
try:
|
||||
manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return directory, [], "private manifest.json is missing or unreadable"
|
||||
entries = manifest.get("fixtures") if isinstance(manifest, dict) else None
|
||||
if not isinstance(entries, list) or not entries:
|
||||
return directory, [], "private manifest declares no fixtures"
|
||||
return directory, [entry for entry in entries if isinstance(entry, dict)], None
|
||||
try:
|
||||
if not REPO_EXAMPLES.is_dir():
|
||||
return None, [], None
|
||||
manifest = json.loads((REPO_EXAMPLES / "manifest.json").read_text(encoding="utf-8"))
|
||||
entries = manifest.get("fixtures") if isinstance(manifest, dict) else None
|
||||
if not isinstance(entries, list):
|
||||
return None, [], None
|
||||
return REPO_EXAMPLES, [entry for entry in entries if isinstance(entry, dict)], None
|
||||
except Exception:
|
||||
return None, [], None
|
||||
|
||||
|
||||
FIXTURE_DIR, ENTRIES, EXPLICIT_ERROR = _load_entries()
|
||||
if EXPLICIT_ERROR:
|
||||
raise pytest.fail.Exception(EXPLICIT_ERROR)
|
||||
PARAMS = ENTRIES if ENTRIES else [None]
|
||||
|
||||
|
||||
def _check(ok: bool, name: str, what: str) -> None:
|
||||
assert ok, f"{name}: {what} mismatch"
|
||||
|
||||
|
||||
def _fail(name: str, message: str) -> None:
|
||||
pytest.fail(f"{name}: {message}", pytrace=False)
|
||||
|
||||
|
||||
@pytest.fixture(params=PARAMS, ids=[entry.get("filename", "unavailable") for entry in PARAMS])
|
||||
def parsed(request: pytest.FixtureRequest):
|
||||
entry = request.param
|
||||
if entry is None:
|
||||
pytest.skip("private fixtures are not available")
|
||||
name = str(entry.get("filename", "unnamed"))
|
||||
path = (FIXTURE_DIR / name) if FIXTURE_DIR is not None else None
|
||||
if path is None or not path.is_file():
|
||||
_fail(name, "fixture file is missing")
|
||||
raw = path.read_bytes()
|
||||
kind = magic_kind(raw)
|
||||
if kind not in {"xlsx", "pdf"}:
|
||||
_fail(name, "unexpected file type")
|
||||
try:
|
||||
adapter = detect_adapter(raw, kind)
|
||||
except Exception:
|
||||
_fail(name, "adapter detection failed")
|
||||
if adapter.adapter_id != entry.get("adapter_id"):
|
||||
_fail(name, "adapter detection mismatch")
|
||||
try:
|
||||
first = adapter.parse(raw)
|
||||
second = adapter.parse(raw)
|
||||
except Exception:
|
||||
_fail(name, "parse failed")
|
||||
if first != second:
|
||||
_fail(name, "parse result is not deterministic")
|
||||
return name, entry, first
|
||||
|
||||
|
||||
def test_manifest_contract(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
_check(draft.parser_version == entry.get("parser_version"), name, "parser version")
|
||||
for key, actual in (("period_start", draft.period_start), ("period_end", draft.period_end)):
|
||||
expected = entry.get(key)
|
||||
_check(
|
||||
expected is None or (actual is not None and actual.isoformat() == expected), name, key
|
||||
)
|
||||
|
||||
|
||||
def test_row_counts(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
rows = draft.rows
|
||||
_check(len(rows) == entry.get("row_count"), name, "row count")
|
||||
posted = sum(1 for row in rows if row.bank_status == "posted")
|
||||
holds = sum(1 for row in rows if row.bank_status == "hold")
|
||||
_check(posted == entry.get("posted_count"), name, "posted count")
|
||||
_check(holds == entry.get("hold_count"), name, "hold count")
|
||||
_check(
|
||||
sum(1 for row in rows if row.direction == "in") == entry.get("in_count"),
|
||||
name,
|
||||
"inflow row count",
|
||||
)
|
||||
_check(
|
||||
sum(1 for row in rows if row.direction == "out") == entry.get("out_count"),
|
||||
name,
|
||||
"outflow row count",
|
||||
)
|
||||
|
||||
|
||||
def test_row_invariants(parsed) -> None:
|
||||
name, _, draft = parsed
|
||||
rows = draft.rows
|
||||
_check(all(row.amount_kopecks >= 0 for row in rows), name, "positive amount invariant")
|
||||
_check(
|
||||
all(row.bank_status in {"posted", "hold"} for row in rows),
|
||||
name,
|
||||
"bank status invariant",
|
||||
)
|
||||
|
||||
|
||||
def test_hold_totals(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold")
|
||||
_check(hold_total == entry.get("hold_total_kopecks"), name, "hold total")
|
||||
|
||||
|
||||
def test_control_totals(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
_check(
|
||||
draft.declared_inflow_kopecks == entry.get("declared_inflow_kopecks"),
|
||||
name,
|
||||
"declared inflow total",
|
||||
)
|
||||
_check(
|
||||
draft.declared_outflow_kopecks == entry.get("declared_outflow_kopecks"),
|
||||
name,
|
||||
"declared outflow total",
|
||||
)
|
||||
_check(
|
||||
draft.declared_hold_kopecks == entry.get("declared_hold_kopecks"),
|
||||
name,
|
||||
"declared hold total",
|
||||
)
|
||||
|
||||
|
||||
def test_balances(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
_check(
|
||||
draft.opening_balance_kopecks == entry.get("opening_balance_kopecks"),
|
||||
name,
|
||||
"opening balance",
|
||||
)
|
||||
_check(
|
||||
draft.closing_balance_kopecks == entry.get("closing_balance_kopecks"),
|
||||
name,
|
||||
"closing balance",
|
||||
)
|
||||
|
||||
|
||||
def test_declared_totals_match_parsed_rows(parsed) -> None:
|
||||
name, _, draft = parsed
|
||||
posted_in = sum(
|
||||
row.amount_kopecks
|
||||
for row in draft.rows
|
||||
if row.bank_status == "posted" and row.direction == "in"
|
||||
)
|
||||
posted_out = sum(
|
||||
row.amount_kopecks
|
||||
for row in draft.rows
|
||||
if row.bank_status == "posted" and row.direction == "out"
|
||||
)
|
||||
hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold")
|
||||
# Declared controls are canonicalized to posted-only by the adapters; the
|
||||
# hold control equals the parsed hold sum.
|
||||
_check(posted_in == draft.declared_inflow_kopecks, name, "declared vs parsed inflow")
|
||||
_check(posted_out == draft.declared_outflow_kopecks, name, "declared vs parsed outflow")
|
||||
_check(hold_total == (draft.declared_hold_kopecks or 0), name, "declared vs parsed hold")
|
||||
|
||||
|
||||
def test_service_reconciliation_matches(parsed) -> None:
|
||||
name, _, draft = parsed
|
||||
summary = _reconciliation_summary(draft)
|
||||
_check(summary.get("matched") is True, name, "service reconciliation matched")
|
||||
_check(
|
||||
summary.get("posted_totals_matched") is True,
|
||||
name,
|
||||
"service reconciliation posted totals",
|
||||
)
|
||||
_check(
|
||||
summary.get("hold_totals_matched") is not False,
|
||||
name,
|
||||
"service reconciliation hold totals",
|
||||
)
|
||||
Reference in New Issue
Block a user