Implement Gitea CI and real statement imports
Gitea Actions CI / detect (push) Successful in 3s
Gitea Actions CI / Compose & nginx validation (push) Failing after 20s
Gitea Actions CI / Frontend (lint / type / test / build) (push) Failing after 24s
Gitea Actions CI / Backend (lint / type / test / build) (push) Failing after 1m34s

This commit is contained in:
k1nq
2026-09-18 17:59:02 +05:00
parent c85b12b95e
commit e43428c1a2
25 changed files with 19256 additions and 6208 deletions
@@ -0,0 +1,133 @@
"""Hold lifecycle: source bank status and append-only hold resolutions."""
import sqlalchemy as sa
from alembic import op
revision = "0002"
down_revision = "0001"
branch_labels = None
depends_on = None
def _has_column(bind, table: str, column: str) -> bool:
inspector = sa.inspect(bind)
return column in {item["name"] for item in inspector.get_columns(table)}
def _has_check_constraint(bind, table: str, suffix: str) -> bool:
inspector = sa.inspect(bind)
return any(
(item.get("name") or "").endswith(suffix) for item in inspector.get_check_constraints(table)
)
def upgrade() -> None:
bind = op.get_bind()
# Revision 0001 materializes the schema from live model metadata, so a
# database created fresh already carries these objects; guard every step.
if not _has_column(bind, "source_records", "bank_status"):
op.add_column(
"source_records",
sa.Column("bank_status", sa.String(length=8), nullable=False, server_default="posted"),
)
if bind.dialect.name == "postgresql" and not _has_check_constraint(
bind, "source_records", "bank_status_valid"
):
op.create_check_constraint(
"bank_status_valid",
"source_records",
"bank_status IN ('posted', 'hold')",
)
inspector = sa.inspect(bind)
if not inspector.has_table("hold_resolutions"):
op.create_table(
"hold_resolutions",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("space_id", sa.Uuid(), nullable=False),
sa.Column("hold_source_record_id", sa.Uuid(), nullable=False),
sa.Column("resolution", sa.String(length=24), nullable=False),
sa.Column("resolving_source_record_id", sa.Uuid(), nullable=True),
sa.Column("operation_id", sa.Uuid(), nullable=True),
sa.Column("resolved_by_user_id", sa.Uuid(), nullable=True),
sa.Column("reason", sa.String(length=500), nullable=True),
sa.Column("resolved_at", sa.DateTime(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
sa.CheckConstraint(
"resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')",
name="ck_hold_resolutions_resolution_valid",
),
sa.CheckConstraint(
"NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') "
"AND (reason IS NULL OR reason = ''))",
name="ck_hold_resolutions_manual_reason_required",
),
sa.ForeignKeyConstraint(
["space_id"],
["financial_spaces.id"],
ondelete="CASCADE",
name="fk_hold_resolutions_space_id_financial_spaces",
),
sa.ForeignKeyConstraint(
["space_id", "hold_source_record_id"],
["source_records.space_id", "source_records.id"],
ondelete="CASCADE",
name="fk_hold_resolutions_hold_tenant",
),
sa.ForeignKeyConstraint(
["space_id", "resolving_source_record_id"],
["source_records.space_id", "source_records.id"],
name="fk_hold_resolutions_resolving_tenant",
),
sa.ForeignKeyConstraint(
["space_id", "operation_id"],
["operations.space_id", "operations.id"],
name="fk_hold_resolutions_operation_tenant",
),
sa.ForeignKeyConstraint(
["resolved_by_user_id"],
["users.id"],
ondelete="SET NULL",
name="fk_hold_resolutions_resolved_by_user_id_users",
),
sa.PrimaryKeyConstraint("id", name="pk_hold_resolutions"),
sa.UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"),
sa.UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"),
)
op.create_index("ix_hold_resolutions_space_id", "hold_resolutions", ["space_id"])
op.create_index(
"ix_hold_resolutions_hold_source_record_id",
"hold_resolutions",
["hold_source_record_id"],
)
op.create_index("ix_hold_resolutions_operation_id", "hold_resolutions", ["operation_id"])
if bind.dialect.name == "postgresql":
# Revision 0001 creates tables from live metadata, so a fresh database
# already has hold_resolutions before this revision runs. RLS setup must
# therefore be independent of the create-table branch above.
op.execute("ALTER TABLE hold_resolutions ENABLE ROW LEVEL SECURITY")
op.execute("ALTER TABLE hold_resolutions FORCE ROW LEVEL SECURITY")
op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions")
op.execute(
"CREATE POLICY tenant_isolation ON hold_resolutions "
"USING (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid) "
"WITH CHECK (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid)"
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("hold_resolutions"):
if bind.dialect.name == "postgresql":
op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions")
op.drop_index("ix_hold_resolutions_operation_id", table_name="hold_resolutions")
op.drop_index("ix_hold_resolutions_hold_source_record_id", table_name="hold_resolutions")
op.drop_index("ix_hold_resolutions_space_id", table_name="hold_resolutions")
op.drop_table("hold_resolutions")
if _has_column(bind, "source_records", "bank_status"):
with op.batch_alter_table("source_records") as batch_op:
if _has_check_constraint(bind, "source_records", "bank_status_valid"):
batch_op.drop_constraint("bank_status_valid", type_="check")
batch_op.drop_column("bank_status")
+23 -1
View File
@@ -440,7 +440,7 @@ def _close_blockers(
uncategorized_count: int,
negative_envelopes: list[str],
) -> list[dict]:
from app.imports.models import StatementDocument
from app.imports.models import HoldResolution, SourceRecord, StatementDocument
blockers = []
unresolved = (
@@ -455,6 +455,28 @@ def _close_blockers(
)
if unresolved:
blockers.append({"code": "IMPORTS_UNRESOLVED", "count": unresolved})
start, end = period_bounds(period.year, period.month)
unresolved_holds = (
db.query(SourceRecord.id)
.join(
StatementDocument,
(StatementDocument.space_id == SourceRecord.space_id)
& (StatementDocument.id == SourceRecord.document_id),
)
.outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id)
.filter(
SourceRecord.space_id == space_id,
SourceRecord.bank_status == "hold",
StatementDocument.status == "confirmed",
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
HoldResolution.id.is_(None),
SourceRecord.occurred_on >= start,
SourceRecord.occurred_on < end,
)
.count()
)
if unresolved_holds:
blockers.append({"code": "IMPORT_UNRESOLVED_HOLDS", "count": unresolved_holds})
if uncategorized_count:
blockers.append({"code": "BUDGET_UNCATEGORIZED_EXPENSES", "count": uncategorized_count})
if negative_envelopes:
+8 -4
View File
@@ -1,13 +1,17 @@
from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter
from app.imports.adapters.alfa_xlsx import (
AlfaCreditXlsxAdapter as AlfaCreditXlsxAdapter,
)
from app.imports.adapters.alfa_xlsx import AlfaDebitXlsxAdapter
from app.imports.adapters.base import StatementAdapter, UnsupportedStatementLayout
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter as OzonCreditPdfAdapter
from app.imports.adapters.ozon_pdf import OzonDebitPdfAdapter
from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter
# Only the three evidenced-format development contracts are active. The credit
# adapters stay present but unregistered until real fixtures exist for them.
ADAPTERS: tuple[StatementAdapter, ...] = (
AlfaDebitXlsxAdapter(),
AlfaCreditXlsxAdapter(),
OzonDebitPdfAdapter(),
OzonCreditPdfAdapter(),
RocketbankDebitPdfAdapter(),
)
+279 -2
View File
@@ -1,9 +1,16 @@
from __future__ import annotations
import re
from datetime import date, datetime
from typing import TypedDict
from openpyxl.workbook import Workbook
from app.errors import AppError
from app.imports.adapters.base import (
ALFA_DEBIT_XLSX_VERSION,
BANK_STATUS_HOLD,
BANK_STATUS_POSTED,
DIRECTION_IN,
DIRECTION_OUT,
PARSER_VERSION,
@@ -12,6 +19,7 @@ from app.imports.adapters.base import (
UnsupportedStatementLayout,
account_fingerprint,
amount_to_kopecks,
money_to_kopecks,
)
from app.imports.readers import open_xlsx
@@ -198,10 +206,279 @@ def _find_account_value(rows: list[tuple]) -> str | None:
return masked
class AlfaDebitXlsxAdapter(AlfaXlsxAdapterBase):
class AlfaDebitXlsxAdapter:
"""Alfa debit statement contract, evidenced against the real export layout.
Workbook shape: a 'Выписка по счету' title block, account metadata with
the exact statement period ('За период с ... по ...') and header control
totals (Поступления / Расходы / Неподтвержденные операции / Текущий
баланс), an operation header row (Дата операции, Дата проводки, Код,
Категория, Описание, Сумма в валюте счета, Статус), then operation rows
terminated by the signature footer. A HOLD posting-date cell marks an
immutable hold row; the Код column becomes the external id; a trailing
MCC token in the description becomes the row MCC. The raw Расходы control
includes hold amounts, so declared_outflow_kopecks is canonicalized to
posted-only (raw outflow minus declared hold). Anything else fails closed.
"""
adapter_id = "alfa_debit_xlsx"
product = "debit"
product_markers = ("дебет", "дебетовая")
parser_version = ALFA_DEBIT_XLSX_VERSION
bank_code = "alfa"
currency = "RUB"
detect_scan_rows = 60
header_scan_rows = 30
title_label = "выписка по счету"
period_re = re.compile(
r"за период с\s*(\d{2}\.\d{2}\.\d{4})\s*по\s*(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE
)
mcc_re = re.compile(r"\bMCC\s*:?\s*(\d{4})\b", re.IGNORECASE)
header_labels = {
"date": "дата операции",
"posted": "дата проводки",
"code": "код",
"category": "категория",
"description": "описание",
"status": "статус",
}
amount_label_prefix = "сумма в"
meta_labels = {
"account": "номер счета",
"currency": "валюта счета",
"inflow": "поступления",
"outflow": "расходы",
"hold": "неподтвержденные операции",
"closing": "текущий баланс",
}
footer_markers = ("подпись сотрудника", "уполномоченное лицо")
footer_page_prefix = "страница"
hold_token = "hold"
def detect(self, raw: bytes) -> bool:
try:
with open_xlsx(raw) as workbook:
rows = _debit_read_rows(workbook, self.detect_scan_rows)
except Exception:
return False
cells = [_debit_norm(cell) for row in rows for cell in row]
if self.title_label not in cells:
return False
required = (self.header_labels["date"], self.header_labels["posted"], "статус")
return all(label in cells for label in required)
def parse(self, raw: bytes) -> StatementDraft:
with open_xlsx(raw) as workbook:
cap = get_settings_row_cap()
rows = _debit_read_rows(workbook, cap + 1)
if len(rows) > cap:
raise UnsupportedStatementLayout("row quota exceeded during parse")
header_idx, columns = _debit_find_header(self, rows)
meta = _debit_header_metadata(self, rows, header_idx)
period_start, period_end = _debit_statement_period(self, rows, header_idx)
parsed = _debit_operation_rows(self, rows, header_idx, columns)
if not parsed:
raise UnsupportedStatementLayout("no operation rows found in Alfa statement")
return StatementDraft(
adapter_id=self.adapter_id,
parser_version=self.parser_version,
bank_code=self.bank_code,
product=self.product,
account_fingerprint=account_fingerprint(self.bank_code, meta["account"]),
currency=meta["currency"],
period_start=period_start,
period_end=period_end,
opening_balance_kopecks=None,
closing_balance_kopecks=meta["closing"],
rows=tuple(parsed),
declared_inflow_kopecks=meta["inflow"],
declared_outflow_kopecks=meta["outflow"],
declared_hold_kopecks=meta["hold"],
)
def _debit_norm(cell: object) -> str:
if cell is None:
return ""
return str(cell).replace("\u00a0", " ").strip().lower()
def _debit_read_rows(workbook: Workbook, limit: int) -> list[tuple]:
sheet = workbook.worksheets[0]
rows: list[tuple] = []
for index, row in enumerate(sheet.iter_rows(values_only=True)):
if index >= limit:
break
rows.append(row)
return rows
def _debit_find_header(
adapter: AlfaDebitXlsxAdapter, rows: list[tuple]
) -> tuple[int, dict[str, int]]:
for index, row in enumerate(rows[: adapter.header_scan_rows]):
cells = [_debit_norm(cell) for cell in row]
columns: dict[str, int] = {}
for key, label in adapter.header_labels.items():
column = next((i for i, cell in enumerate(cells) if cell == label), None)
if column is None:
break
columns[key] = column
if len(columns) != len(adapter.header_labels):
continue
amount_column = next(
(i for i, cell in enumerate(cells) if cell.startswith(adapter.amount_label_prefix)),
None,
)
if amount_column is None:
continue
columns["amount"] = amount_column
return index, columns
raise UnsupportedStatementLayout("Alfa operation header row not found")
def _debit_header_value(rows: list[tuple], label: str) -> str:
for row in rows:
for index, cell in enumerate(row):
if _debit_norm(cell) != label:
continue
for candidate in row[index + 1 :]:
text = _cell_str(candidate)
if text:
return text
raise UnsupportedStatementLayout(f"Alfa header metadata not found: {label}")
class _AlfaHeaderMeta(TypedDict):
account: str
currency: str
inflow: int
outflow: int
hold: int
closing: int
def _debit_statement_period(
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
) -> tuple[date, date]:
text = "\n".join(_debit_norm(cell) for row in rows[:header_idx] for cell in row)
period_match = adapter.period_re.search(text)
if period_match is None:
raise UnsupportedStatementLayout("Alfa statement period not found")
period_start = _parse_date(period_match.group(1))
period_end = _parse_date(period_match.group(2))
if period_start is None or period_end is None:
raise UnsupportedStatementLayout("Alfa statement period not found")
return period_start, period_end
def _debit_header_metadata(
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
) -> _AlfaHeaderMeta:
meta: _AlfaHeaderMeta = {
"account": _debit_header_value(rows[:header_idx], adapter.meta_labels["account"]),
"currency": "RUB",
"inflow": 0,
"outflow": 0,
"hold": 0,
"closing": 0,
}
currency = _debit_header_value(rows[:header_idx], adapter.meta_labels["currency"])
if currency.replace(" ", "").upper() not in {"RUB", "RUR"}:
raise UnsupportedStatementLayout("unsupported Alfa statement currency")
raw_outflow = money_to_kopecks(
_debit_header_value(rows[:header_idx], adapter.meta_labels["outflow"])
)
meta["hold"] = money_to_kopecks(
_debit_header_value(rows[:header_idx], adapter.meta_labels["hold"])
)
# The raw expense control includes pending (hold) amounts; canonicalize to
# posted-only so service reconciliation can compare like with like.
canonical_outflow = raw_outflow - meta["hold"]
if canonical_outflow < 0:
raise UnsupportedStatementLayout("inconsistent Alfa expense header controls")
meta["outflow"] = canonical_outflow
meta["inflow"] = money_to_kopecks(
_debit_header_value(rows[:header_idx], adapter.meta_labels["inflow"])
)
meta["closing"] = money_to_kopecks(
_debit_header_value(rows[:header_idx], adapter.meta_labels["closing"])
)
return meta
def _debit_operation_rows(
adapter: AlfaDebitXlsxAdapter,
rows: list[tuple],
header_idx: int,
columns: dict[str, int],
) -> list[SourceRow]:
parsed: list[SourceRow] = []
separated_from_operations = False
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 2):
norm_cells = [_debit_norm(cell) for cell in row]
if any(marker in cell for cell in norm_cells for marker in adapter.footer_markers):
break
if any(cell.startswith(adapter.footer_page_prefix) for cell in norm_cells):
break
if all(cell == "" for cell in norm_cells):
separated_from_operations = bool(parsed)
continue
occurred = _parse_date(row[columns["date"]])
posted_cell = _cell_str(row[columns["posted"]]) or ""
status_cell = _cell_str(row[columns["status"]]) or ""
bank_status = BANK_STATUS_POSTED
if adapter.hold_token in posted_cell.lower() or adapter.hold_token in status_cell.lower():
bank_status = BANK_STATUS_HOLD
posted_on = None
else:
posted_on = _parse_date(posted_cell)
amount_text = _cell_str(row[columns["amount"]])
amount: int | None = None
if amount_text is not None:
try:
amount = abs(money_to_kopecks(amount_text))
except UnsupportedStatementLayout:
amount = None
if occurred is None and amount is None:
later_has_operation = any(
_parse_date(later[columns["date"]]) is not None
for later in rows[offset:]
if len(later) > columns["date"]
)
if separated_from_operations and not later_has_operation:
break
raise UnsupportedStatementLayout("unexpected row in Alfa operation table")
warnings: list[str] = []
if occurred is None:
warnings.append("occurred_date_missing")
if amount is None:
amount = 0
warnings.append("amount_unparseable")
if bank_status == BANK_STATUS_POSTED and posted_on is None and posted_cell:
warnings.append("posted_date_unparseable")
direction = DIRECTION_OUT if (amount_text or "").startswith("-") else DIRECTION_IN
description = _cell_str(row[columns["description"]])
mcc_match = adapter.mcc_re.search(description) if description else None
parsed.append(
SourceRow(
external_id=_cell_str(row[columns["code"]]),
occurred_on=occurred,
posted_on=posted_on,
amount_kopecks=amount,
direction=direction,
description=description,
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
bank_category=_cell_str(row[columns["category"]]),
location=None,
row_ref=f"sheet0:row{offset}",
warnings=tuple(warnings),
bank_status=bank_status,
)
)
separated_from_operations = False
return parsed
class AlfaCreditXlsxAdapter(AlfaXlsxAdapterBase):
+45
View File
@@ -1,4 +1,5 @@
import hashlib
import re
from dataclasses import dataclass
from datetime import date
from decimal import Decimal, InvalidOperation
@@ -8,7 +9,16 @@ from app.errors import AppError
DIRECTION_IN = "in"
DIRECTION_OUT = "out"
BANK_STATUS_POSTED = "posted"
BANK_STATUS_HOLD = "hold"
BANK_STATUSES = (BANK_STATUS_POSTED, BANK_STATUS_HOLD)
# Legacy shared version kept only for the untouched, unregistered credit adapters.
PARSER_VERSION = "1"
# Per-contract versions: bumped for the rewritten evidenced contracts so old
# parse runs can never be confused with the new canonical output.
ALFA_DEBIT_XLSX_VERSION = "2"
OZON_DEBIT_PDF_VERSION = "2"
ROCKETBANK_DEBIT_PDF_VERSION = "2"
@dataclass(frozen=True)
@@ -24,6 +34,15 @@ class SourceRow:
location: str | None
row_ref: str
warnings: tuple[str, ...] = ()
bank_status: str = BANK_STATUS_POSTED
def __post_init__(self) -> None:
if self.amount_kopecks < 0:
raise UnsupportedStatementLayout(
"amount_kopecks must be positive; direction carries the sign"
)
if self.bank_status not in BANK_STATUSES:
raise UnsupportedStatementLayout("bank_status must be posted or hold")
@dataclass(frozen=True)
@@ -39,6 +58,9 @@ class StatementDraft:
opening_balance_kopecks: int | None
closing_balance_kopecks: int | None
rows: tuple[SourceRow, ...]
declared_inflow_kopecks: int | None = None
declared_outflow_kopecks: int | None = None
declared_hold_kopecks: int | None = None
class UnsupportedStatementLayout(AppError):
@@ -77,6 +99,29 @@ def amount_to_kopecks(value: object) -> int:
raise UnsupportedStatementLayout(f"unsupported amount cell type: {type(value).__name__}")
def _mask_digits(text: str) -> str:
return re.sub(r"\d", "#", text)
CURRENCY_SUFFIX_RE = re.compile(r"(?:₽|RUR|rub|руб\.?)\s*$", re.IGNORECASE)
PLAIN_AMOUNT_RE = re.compile(r"^[+-]?\s?\d[\d\s]*(?:[.,]\d{1,2})?$")
def money_to_kopecks(text: object) -> int:
"""Parse statement money strings like '-1 234,56 RUR', '+ 500.00 ₽', '12 345'.
Accepts space/NBSP thousand separators and comma or dot decimal marks.
Fails closed with a digit-masked diagnostic; never leaks the raw value.
"""
if not isinstance(text, str):
raise UnsupportedStatementLayout(f"unsupported money cell type: {type(text).__name__}")
cleaned = text.replace("\u00a0", " ").strip()
cleaned = CURRENCY_SUFFIX_RE.sub("", cleaned).strip()
if not PLAIN_AMOUNT_RE.fullmatch(cleaned):
raise UnsupportedStatementLayout(f"unparseable money amount: {_mask_digits(text)}")
return amount_to_kopecks(cleaned)
def account_fingerprint(bank_code: str, masked_identifier: str) -> str:
digest = hashlib.sha256(f"{bank_code}|{masked_identifier}".encode()).hexdigest()
return digest[:64]
+173 -2
View File
@@ -6,12 +6,14 @@ from datetime import date, datetime
from app.imports.adapters.base import (
DIRECTION_IN,
DIRECTION_OUT,
OZON_DEBIT_PDF_VERSION,
PARSER_VERSION,
SourceRow,
StatementDraft,
UnsupportedStatementLayout,
account_fingerprint,
amount_to_kopecks,
money_to_kopecks,
)
from app.imports.readers import open_pdf
@@ -122,10 +124,179 @@ def _first_page_text(raw: bytes) -> str:
return pdf.pages[0].extract_text() or ""
class OzonDebitPdfAdapter(OzonPdfAdapterBase):
class OzonDebitPdfAdapter:
"""Ozon debit statement contract, evidenced against the real document.
A 'Справка о движении средств' text-layer PDF: a legal-bank header block
with account/period/opening balance, per-page tables whose rows keep
multiline document IDs and descriptions attached to one transaction, and a
closing block with declared inflow/outflow totals and the closing balance.
Amounts are signed strings ('- 500.00 ₽'); the sign maps to direction and
the stored amount is always positive.
"""
adapter_id = "ozon_debit_pdf"
product = "debit"
product_markers = ("дебетов",)
parser_version = OZON_DEBIT_PDF_VERSION
bank_code = "ozon"
currency = "RUB"
bank_marker = "озон банк"
title_marker = "справка о движении средств"
account_marker_re = re.compile(r"лицевого\s+сч[её]та:?\s*№\s*([\d*•]+)", re.IGNORECASE)
period_re = re.compile(
r"период\s+выписки:?\s*(\d{2}\.\d{2}\.\d{4})\s*[–—-]\s*(\d{2}\.\d{2}\.\d{4})",
re.IGNORECASE,
)
opening_re = re.compile(r"входящий\s+остаток:?\s*(.+)", re.IGNORECASE)
inflow_re = re.compile(r"итого\s+зачислений\s+за\s+период:?\s*(.+)", re.IGNORECASE)
outflow_re = re.compile(r"итого\s+списаний\s+за\s+период:?\s*(.+)", re.IGNORECASE)
closing_re = re.compile(r"исходящий\s+остаток:?\s*(.+)", re.IGNORECASE)
amount_re = re.compile(r"^([+-])\s*(\d[\d\s]*(?:[.,]\d{2})?)\s*(?:₽|rur|rub)?$", re.IGNORECASE)
table_header_labels = ("дата операции", "документ", "назначение платежа")
table_subheader_tokens = ("рубли", "валюта")
def detect(self, raw: bytes) -> bool:
try:
text = _first_page_text(raw)
except Exception:
return False
lowered = _ozon_normalize(text)
return (
self.bank_marker in lowered
and self.title_marker in lowered
and "лицевого сч" in lowered
)
def parse(self, raw: bytes) -> StatementDraft:
pages_text: list[str] = []
pages_tables: list[list[list[list[object]]]] = []
with open_pdf(raw) as pdf:
if not pdf.pages:
raise UnsupportedStatementLayout("PDF has no pages")
for page_index, page in enumerate(pdf.pages):
text = page.extract_text() or ""
if page_index == 0 and not text.strip():
raise UnsupportedStatementLayout(
"PDF has no extractable text layer; OCR is not supported"
)
pages_text.append(text)
extracted = page.extract_tables()
pages_tables.append(extracted if extracted else [])
return self._draft_from_extracted(pages_text, pages_tables)
def _draft_from_extracted(
self, pages_text: list[str], pages_tables: list[list[list[list[object]]]]
) -> StatementDraft:
page_zero = _ozon_normalize(pages_text[0])
if self.bank_marker not in page_zero or self.title_marker not in page_zero:
raise UnsupportedStatementLayout("Ozon statement markers not found")
account_match = self.account_marker_re.search(page_zero)
if account_match is None:
raise UnsupportedStatementLayout("Ozon account line not found")
period_match = self.period_re.search(page_zero)
if period_match is None:
raise UnsupportedStatementLayout("Ozon period line not found")
period_start = datetime.strptime(period_match.group(1), "%d.%m.%Y").date()
period_end = datetime.strptime(period_match.group(2), "%d.%m.%Y").date()
opening = money_to_kopecks(self._required_line(self.opening_re, page_zero, "opening"))
full_text = _ozon_normalize("\n".join(pages_text))
declared_inflow = money_to_kopecks(self._required_line(self.inflow_re, full_text, "inflow"))
declared_outflow = money_to_kopecks(
self._required_line(self.outflow_re, full_text, "outflow")
)
closing = money_to_kopecks(self._required_line(self.closing_re, full_text, "closing"))
rows: list[SourceRow] = []
for page_index, tables in enumerate(pages_tables, start=1):
for table_index, table in enumerate(tables):
for row_index, cells in enumerate(table):
row = self._table_row(
cells, f"page{page_index}:table{table_index}:row{row_index}"
)
if row is not None:
rows.append(row)
if not rows:
raise UnsupportedStatementLayout("no statement rows matched the Ozon contract")
return StatementDraft(
adapter_id=self.adapter_id,
parser_version=self.parser_version,
bank_code=self.bank_code,
product=self.product,
account_fingerprint=account_fingerprint(self.bank_code, account_match.group(1)),
currency=self.currency,
period_start=period_start,
period_end=period_end,
opening_balance_kopecks=opening,
closing_balance_kopecks=closing,
rows=tuple(rows),
declared_inflow_kopecks=declared_inflow,
declared_outflow_kopecks=declared_outflow,
declared_hold_kopecks=None,
)
def _required_line(self, pattern: re.Pattern[str], text: str, what: str) -> str:
match = pattern.search(text)
if match is None:
raise UnsupportedStatementLayout(f"Ozon {what} total line not found")
return match.group(1)
def _table_row(self, cells: list[object], row_ref: str) -> SourceRow | None:
if cells is None:
raise UnsupportedStatementLayout("unexpected Ozon table row shape")
texts = [_ozon_cell(cell) for cell in cells]
if not any(texts):
return None
lowered = [text.lower() for text in texts]
joined = " ".join(text for text in lowered if text)
if all(label in joined for label in self.table_header_labels):
return None
if lowered[0] == "" and any(token in joined for token in self.table_subheader_tokens):
return None
if lowered[0] == "" or len(texts) < 4:
raise UnsupportedStatementLayout("unexpected Ozon table row shape")
occurred = None
date_match = re.match(r"^(\d{2}\.\d{2}\.\d{4})", texts[0])
if date_match is not None:
occurred = datetime.strptime(date_match.group(1), "%d.%m.%Y").date()
external_id = re.sub(r"\s+", "", texts[1]) if len(texts) > 1 else ""
description = " ".join(texts[2].split()) if len(texts) > 2 else ""
warnings: list[str] = []
if occurred is None:
warnings.append("occurred_date_missing")
if not external_id:
warnings.append("external_id_missing")
amount_match = self.amount_re.match(texts[3]) if len(texts) > 3 else None
if amount_match is None:
amount = 0
warnings.append("amount_unparseable")
direction = DIRECTION_OUT
else:
sign, body = amount_match.group(1), amount_match.group(2)
amount = amount_to_kopecks(body.replace(" ", "").replace(",", "."))
direction = DIRECTION_IN if sign == "+" else DIRECTION_OUT
return SourceRow(
external_id=external_id or None,
occurred_on=occurred,
posted_on=None,
amount_kopecks=amount,
direction=direction,
description=description or None,
mcc=None,
bank_category=None,
location=None,
row_ref=row_ref,
warnings=tuple(warnings),
)
def _ozon_normalize(text: str) -> str:
return text.replace("\u00a0", " ").lower()
def _ozon_cell(cell: object) -> str:
if cell is None:
return ""
return str(cell).replace("\u00a0", " ").strip()
class OzonCreditPdfAdapter(OzonPdfAdapterBase):
+281 -5
View File
@@ -1,12 +1,288 @@
from __future__ import annotations
from app.imports.adapters.ozon_pdf import OzonPdfAdapterBase
import re
from datetime import date
from app.imports.adapters.base import (
DIRECTION_IN,
DIRECTION_OUT,
ROCKETBANK_DEBIT_PDF_VERSION,
SourceRow,
StatementDraft,
UnsupportedStatementLayout,
account_fingerprint,
money_to_kopecks,
)
from app.imports.readers import open_pdf
MONTH_NAMES = {
"января": 1,
"февраля": 2,
"марта": 3,
"апреля": 4,
"мая": 5,
"июня": 6,
"июля": 7,
"августа": 8,
"сентября": 9,
"октября": 10,
"ноября": 11,
"декабря": 12,
}
DAY_HEADING_RE = re.compile(
r"^(\d{1,2})\s+(января|февраля|марта|апреля|мая|июня|июля|августа|сентября|"
r"октября|ноября|декабря)\s+(\d{4})\s+баланс на начало дня:\s*(-?[\d\s]+,\d{2})\s*rur$",
re.IGNORECASE,
)
RECORD_RE = re.compile(
r"^(\d{1,2}:\d{2})\s+(\d{6,})\s+(.*?)([+-]?\s?[\d\s]+,\d{2})\s*rur$",
re.IGNORECASE,
)
HEADER_VALUES_RE = re.compile(
r"^(-?[\d\s]+,\d{2})\s*rur\s+(-?[\d\s]+,\d{2})\s*rur(?:\s+(\d{2}\.\d{2}\.\d{4}))?$",
re.IGNORECASE,
)
PERIOD_RE = re.compile(r"^с\s+(\d{2}\.\d{2}\.\d{4})\s+по\s+(\d{2}\.\d{2}\.\d{4})$", re.IGNORECASE)
AMOUNT_RE = re.compile(r"^([+-]?)\s*(\d[\d\s]*,\d{2})$")
MCC_RE = re.compile(r"\bmcc\s*:?\s*(\d{4})\b", re.IGNORECASE)
LOCATION_RE = re.compile(r"rur,\s*([^,]+?),\s*mcc\b", re.IGNORECASE)
class RocketbankDebitPdfAdapter(OzonPdfAdapterBase):
"""Rocketbank debit statement contract: same text-layer line scanning shape."""
class RocketbankDebitPdfAdapter:
"""Rocketbank debit statement contract, evidenced against the real document.
Independent text-layer state machine: every page repeats the statement
header and column captions; day sections start with a Russian day heading
('DD MONTH YYYY БАЛАНС НА НАЧАЛО ДНЯ: X RUR'); each record is a
'time + operation code + multiline description + signed RUR amount' line
whose description column starts slightly above and continues below that
line, so the surrounding non-structural lines are folded into one
description. Daily totals and repeated captions are ignored. Statement
period, account, opening/closing balances and declared inflow/outflow
totals come from the first-page header block.
"""
adapter_id = "rocketbank_debit_pdf"
product = "debit"
bank_markers = ("рокетбанк", "rocketbank")
product_markers = ("дебетов", "дебетовая")
parser_version = ROCKETBANK_DEBIT_PDF_VERSION
bank_code = "rocketbank"
currency = "RUB"
title_marker = "выписка по счету"
card_re = re.compile(r"дебетовая карта рокет\s*(\d{10,30})\s*rur", re.IGNORECASE)
column_caption = "время код операции описание сумма"
issued_prefix = "дата выдачи выписки:"
page_footer_prefix = "страница"
daily_total_prefixes = ("всего поступлений:", "всего расходов:")
opening_label = "входящий остаток"
closing_label = "исходящий остаток"
max_fold_lines = 6
def detect(self, raw: bytes) -> bool:
try:
with open_pdf(raw) as pdf:
if not pdf.pages:
return False
text = pdf.pages[0].extract_text() or ""
except Exception:
return False
lowered = text.lower()
return self.title_marker in lowered and "дебетовая карта рокет" in lowered
def parse(self, raw: bytes) -> StatementDraft:
lines: list[tuple[int, str]] = []
with open_pdf(raw) as pdf:
if not pdf.pages:
raise UnsupportedStatementLayout("PDF has no pages")
for page_number, page in enumerate(pdf.pages, start=1):
text = page.extract_text() or ""
if page_number == 1 and not text.strip():
raise UnsupportedStatementLayout(
"PDF has no extractable text layer; OCR is not supported"
)
for line in text.splitlines():
stripped = line.strip()
if stripped:
lines.append((page_number, stripped))
return self._draft_from_lines(lines)
def _draft_from_lines(self, lines: list[tuple[int, str]]) -> StatementDraft:
blob = "\n".join(line.lower() for _, line in lines)
if self.title_marker not in blob:
raise UnsupportedStatementLayout("Rocketbank statement marker not found")
card_match = self.card_re.search(blob)
if card_match is None:
raise UnsupportedStatementLayout("Rocketbank card line not found")
period_start: date | None = None
period_end: date | None = None
opening: int | None = None
closing: int | None = None
declared_inflow: int | None = None
declared_outflow: int | None = None
rows: list[SourceRow] = []
current_day: date | None = None
pending: _PendingRecord | None = None
pre_lines: list[tuple[int, str]] = []
record_ordinals: dict[int, int] = {}
index = 0
while index < len(lines):
page, line = lines[index]
low = line.lower()
if (
low == self.title_marker
or low.startswith(self.issued_prefix)
or low.startswith(self.page_footer_prefix)
or low == self.column_caption
or low.startswith(self.daily_total_prefixes)
):
index += 1
continue
period_match = PERIOD_RE.match(low)
if period_match is not None and period_start is None:
period_start = _date_from(period_match.group(1))
period_end = _date_from(period_match.group(2))
index += 1
continue
if self.opening_label in low or self.closing_label in low:
if index + 1 >= len(lines):
raise UnsupportedStatementLayout("Rocketbank balance values missing")
values_match = HEADER_VALUES_RE.match(lines[index + 1][1].lower())
if values_match is None:
raise UnsupportedStatementLayout("Rocketbank balance values missing")
first = money_to_kopecks(values_match.group(1))
second = money_to_kopecks(values_match.group(2))
if self.opening_label in low:
opening, declared_inflow = first, second
else:
closing, declared_outflow = first, second
index += 2
continue
day_match = DAY_HEADING_RE.match(low)
if day_match is not None:
if pending is not None:
rows.append(self._record(pending, current_day))
pending = None
pre_lines = []
current_day = date(
int(day_match.group(3)),
MONTH_NAMES[day_match.group(2).lower()],
int(day_match.group(1)),
)
index += 1
continue
record_match = RECORD_RE.match(line)
if record_match is not None:
if pending is not None:
rows.append(self._record(pending, current_day))
ordinal = record_ordinals.get(page, 0) + 1
record_ordinals[page] = ordinal
pending = _PendingRecord(
row_ref=f"page{page}:record{ordinal}",
code=record_match.group(2),
description=record_match.group(3).strip(),
amount_text=record_match.group(4),
pre_lines=pre_lines,
)
pre_lines = []
index += 1
continue
if pending is not None:
pending.post_lines.append((page, line))
index += 1
continue
pre_lines.append((page, line))
if len(pre_lines) > self.max_fold_lines:
raise UnsupportedStatementLayout("unrecognized content between Rocketbank sections")
index += 1
if pending is not None:
rows.append(self._record(pending, current_day))
pre_lines = []
if pre_lines:
raise UnsupportedStatementLayout("unrecognized trailing Rocketbank content")
if (
period_start is None
or opening is None
or closing is None
or declared_inflow is None
or declared_outflow is None
):
raise UnsupportedStatementLayout("Rocketbank statement header block not found")
if not rows:
raise UnsupportedStatementLayout("no Rocketbank operation rows found")
return StatementDraft(
adapter_id=self.adapter_id,
parser_version=self.parser_version,
bank_code=self.bank_code,
product=self.product,
account_fingerprint=account_fingerprint(self.bank_code, card_match.group(1)),
currency=self.currency,
period_start=period_start,
period_end=period_end,
opening_balance_kopecks=opening,
closing_balance_kopecks=closing,
rows=tuple(rows),
declared_inflow_kopecks=declared_inflow,
declared_outflow_kopecks=declared_outflow,
declared_hold_kopecks=None,
)
def _record(
self,
pending: _PendingRecord,
current_day: date | None,
) -> SourceRow:
if current_day is None:
raise UnsupportedStatementLayout("Rocketbank record outside a day section")
chunks = [text for _, text in pending.pre_lines]
chunks.append(pending.description)
chunks.extend(text for _, text in pending.post_lines)
description = " ".join(" ".join(chunk.split()) for chunk in chunks if chunk.strip())
amount_match = AMOUNT_RE.match(" ".join(pending.amount_text.split()))
if amount_match is None:
raise UnsupportedStatementLayout("Rocketbank record amount missing")
sign = amount_match.group(1)
amount = money_to_kopecks(amount_match.group(2))
direction = DIRECTION_OUT if sign == "-" else DIRECTION_IN
mcc_match = MCC_RE.search(description)
location_match = LOCATION_RE.search(description)
return SourceRow(
external_id=pending.code,
occurred_on=current_day,
posted_on=None,
amount_kopecks=amount,
direction=direction,
description=description or None,
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
bank_category=None,
location=location_match.group(1).strip() if location_match is not None else None,
row_ref=pending.row_ref,
warnings=(),
)
class _PendingRecord:
__slots__ = ("row_ref", "code", "description", "amount_text", "pre_lines", "post_lines")
def __init__(
self,
*,
row_ref: str,
code: str,
description: str,
amount_text: str,
pre_lines: list[tuple[int, str]],
) -> None:
self.row_ref = row_ref
self.code = code
self.description = description
self.amount_text = amount_text
self.pre_lines = pre_lines
self.post_lines: list[tuple[int, str]] = []
def _date_from(text: str) -> date:
day, month, year = (int(part) for part in text.split("."))
return date(year, month, day)
+61 -2
View File
@@ -1,5 +1,5 @@
import uuid
from datetime import date
from datetime import date, datetime
from typing import Any
from sqlalchemy import (
@@ -18,7 +18,7 @@ from sqlalchemy import (
)
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base, PkMixin, TimestampMixin
from app.db.base import Base, PkMixin, TimestampMixin, UTCDateTime, utcnow
DOCUMENT_STATUSES = (
"uploaded",
@@ -34,6 +34,9 @@ RUN_STATUSES = ("pending", "running", "succeeded", "failed")
CANDIDATE_KINDS = ("duplicate", "transfer")
CANDIDATE_CONFIDENCES = ("unique", "possible")
CANDIDATE_RESOLUTIONS = ("pending", "is_duplicate", "is_new")
BANK_STATUSES = ("posted", "hold")
HOLD_RESOLUTIONS = ("matched_posted", "confirmed_as_posted", "discarded_as_cancelled")
HOLD_MANUAL_RESOLUTIONS = ("confirmed_as_posted", "discarded_as_cancelled")
class StatementDocument(Base, PkMixin, TimestampMixin):
@@ -99,6 +102,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin):
__tablename__ = "source_records"
__table_args__ = (
CheckConstraint("bank_status IN ('posted', 'hold')", name="bank_status_valid"),
ForeignKeyConstraint(
["space_id", "document_id"],
["statement_documents.space_id", "statement_documents.id"],
@@ -128,6 +132,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin):
posted_on: Mapped[date | None] = mapped_column(Date, nullable=True)
amount_kopecks: Mapped[int] = mapped_column(BigInteger, nullable=False)
direction: Mapped[str] = mapped_column(String(8), nullable=False)
bank_status: Mapped[str] = mapped_column(String(8), nullable=False, default="posted")
description: Mapped[str | None] = mapped_column(String(500), nullable=True)
mcc: Mapped[int | None] = mapped_column(Integer, nullable=True)
bank_category: Mapped[str | None] = mapped_column(String(100), nullable=True)
@@ -209,3 +214,57 @@ class DedupCandidate(Base, PkMixin, TimestampMixin):
decided_by_user_id: Mapped[uuid.UUID | None] = mapped_column(
Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
)
class HoldResolution(Base, PkMixin, TimestampMixin):
"""Append-only, tenant-scoped settlement of one HOLD source record.
The extracted HOLD row itself is never mutated. At most one resolution
exists per hold source record; automatic matches (`matched_posted`) are
created only when a later statement is confirmed, and manual actions
(`confirmed_as_posted`, `discarded_as_cancelled`) always carry a reason.
"""
__tablename__ = "hold_resolutions"
__table_args__ = (
CheckConstraint(
"resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')",
name="resolution_valid",
),
CheckConstraint(
"NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') "
"AND (reason IS NULL OR reason = ''))",
name="manual_reason_required",
),
ForeignKeyConstraint(
["space_id", "hold_source_record_id"],
["source_records.space_id", "source_records.id"],
ondelete="CASCADE",
name="fk_hold_resolutions_hold_tenant",
),
ForeignKeyConstraint(
["space_id", "resolving_source_record_id"],
["source_records.space_id", "source_records.id"],
name="fk_hold_resolutions_resolving_tenant",
),
ForeignKeyConstraint(
["space_id", "operation_id"],
["operations.space_id", "operations.id"],
name="fk_hold_resolutions_operation_tenant",
),
UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"),
UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"),
)
space_id: Mapped[uuid.UUID] = mapped_column(
Uuid, ForeignKey("financial_spaces.id", ondelete="CASCADE"), nullable=False, index=True
)
hold_source_record_id: Mapped[uuid.UUID] = mapped_column(Uuid, nullable=False, index=True)
resolution: Mapped[str] = mapped_column(String(24), nullable=False)
resolving_source_record_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True)
operation_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True, index=True)
resolved_by_user_id: Mapped[uuid.UUID | None] = mapped_column(
Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
)
reason: Mapped[str | None] = mapped_column(String(500), nullable=True)
resolved_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False, default=utcnow)
+66 -16
View File
@@ -33,6 +33,14 @@ class ConfirmBody(BaseModel):
reason: str | None = None
class HoldResolveBody(BaseModel):
model_config = ConfigDict(populate_by_name=True)
action: str
reason: str
account_id: uuid.UUID | None = Field(default=None, alias="accountId")
def summary(document: Any) -> dict:
return {
"id": document.id,
@@ -100,21 +108,34 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
document = service._document(db, space_id=context.space.id, document_id=document_id)
base = summary(document)
reconciliation = raw.get("reconciliation") or {}
rows = [
{
"id": item["id"],
"externalId": item["external_id"],
"occurredOn": item["occurred_on"],
"amountKopecks": item["amount_kopecks"],
"direction": item["direction"],
"description": item["description"],
"payeeHint": None,
"warnings": (item["warnings"] or {}).get("codes", []),
"decision": "pending" if item["critical"] else item["review_decision"],
"duplicateOfOperationId": item["duplicate_operation_id"],
}
for item in raw["records"]
]
rows = []
for item in raw["records"]:
hold = item.get("hold_resolution")
rows.append(
{
"id": item["id"],
"externalId": item["external_id"],
"occurredOn": item["occurred_on"],
"amountKopecks": item["amount_kopecks"],
"direction": item["direction"],
"bankStatus": item.get("bank_status", "posted"),
"description": item["description"],
"payeeHint": None,
"warnings": (item["warnings"] or {}).get("codes", []),
"decision": "pending" if item["critical"] else item["review_decision"],
"duplicateOfOperationId": item["duplicate_operation_id"],
"holdResolution": None
if hold is None
else {
"resolution": hold["resolution"],
"resolvingSourceRecordId": hold["resolving_source_record_id"],
"operationId": hold["operation_id"],
"resolvedByUserId": hold["resolved_by_user_id"],
"reason": hold["reason"],
"resolvedAt": hold["resolved_at"],
},
}
)
blockers = [
{"code": "IMPORT_FIELD_UNRESOLVED", "message": None, "rowId": str(item["id"])}
for item in raw["records"]
@@ -136,7 +157,16 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
"status": status,
"openingBalanceKopecks": reconciliation.get("opening_balance_kopecks"),
"closingBalanceKopecks": reconciliation.get("closing_balance_kopecks"),
"computedClosingKopecks": None,
"computedClosingKopecks": reconciliation.get("computed_closing_kopecks"),
"declaredInflowKopecks": reconciliation.get("declared_inflow_kopecks"),
"declaredOutflowKopecks": reconciliation.get("declared_outflow_kopecks"),
"declaredHoldKopecks": reconciliation.get("declared_hold_kopecks"),
"parsedPostedInflowKopecks": reconciliation.get("parsed_posted_inflow_kopecks"),
"parsedPostedOutflowKopecks": reconciliation.get("parsed_posted_outflow_kopecks"),
"parsedHoldTotalKopecks": reconciliation.get("parsed_hold_total_kopecks"),
"postedTotalsMatched": reconciliation.get("posted_totals_matched"),
"holdTotalsMatched": reconciliation.get("hold_totals_matched"),
"balanceMatched": reconciliation.get("balance_matched"),
"reason": reconciliation.get("reason"),
},
"rows": rows,
@@ -181,6 +211,26 @@ def decide_row(
return detail(document_id, db, context)
@router.post("/{document_id}/rows/{row_id}/resolve")
def resolve_hold(
document_id: uuid.UUID,
row_id: uuid.UUID,
body: HoldResolveBody,
db: Db,
context: Context,
_csrf: Csrf,
) -> dict:
service.resolve_hold(
db,
space_id=context.space.id,
document_id=document_id,
source_record_id=row_id,
user=context.user,
**body.model_dump(by_alias=False),
)
return detail(document_id, db, context)
@router.post("/{document_id}/confirm")
def confirm(
document_id: uuid.UUID,
+330 -21
View File
@@ -9,8 +9,9 @@ from sqlalchemy.orm import Session
from app.audit import service as audit
from app.config import get_settings
from app.db.base import utcnow
from app.db.session import session_scope
from app.errors import AppError, conflict, not_found
from app.errors import AppError, bad_request, conflict, not_found
from app.identity.models import User
from app.imports.adapters import detect_adapter, magic_kind
from app.imports.adapters.base import SourceRow, StatementDraft, record_fingerprint
@@ -18,6 +19,7 @@ from app.imports.blob import get_blob_store, new_object_key
from app.imports.crypto import decrypt_file, encrypt_file
from app.imports.models import (
DedupCandidate,
HoldResolution,
ParseRun,
SourceRecord,
SourceRecordLink,
@@ -31,6 +33,8 @@ from app.ledger.service import (
ensure_period_open_for_date,
)
HOLD_MANUAL_ACTIONS = ("confirm_as_posted", "discard_as_cancelled")
def _document(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID) -> StatementDocument:
document = (
@@ -128,12 +132,16 @@ def _draft_json(draft: StatementDraft) -> dict[str, Any]:
def _source_from_row(
*, space_id: uuid.UUID, document_id: uuid.UUID, run_id: uuid.UUID, row: SourceRow
) -> SourceRecord:
critical_warnings = {"occurred_date_missing", "amount_unparseable"}
critical_warnings = {"occurred_date_missing", "amount_unparseable", "bank_status_invalid"}
warnings = list(row.warnings)
if row.direction not in {"in", "out"}:
warnings.append("direction_invalid")
if row.amount_kopecks == 0:
warnings.append("amount_zero")
bank_status = getattr(row, "bank_status", "posted")
if bank_status not in {"posted", "hold"}:
warnings.append("bank_status_invalid")
bank_status = "posted"
return SourceRecord(
space_id=space_id,
document_id=document_id,
@@ -143,6 +151,7 @@ def _source_from_row(
posted_on=row.posted_on,
amount_kopecks=abs(row.amount_kopecks),
direction=row.direction,
bank_status=bank_status,
description=row.description,
mcc=row.mcc,
bank_category=row.bank_category,
@@ -158,9 +167,66 @@ def _source_from_row(
)
def _reconciliation_summary(draft: Any) -> dict[str, Any]:
"""Reconcile parsed rows against declared bank controls with stable diagnostics.
Posted rows are compared with the declared inflow/outflow controls, HOLD
rows with the declared pending total, and where both balances exist the
balance equation must hold. Every check keeps an explicit tri-state field:
True (matched), False (mismatch, review-blocking), None (not declared).
"""
rows = list(draft.rows)
posted = [row for row in rows if getattr(row, "bank_status", "posted") == "posted"]
holds = [row for row in rows if getattr(row, "bank_status", "posted") == "hold"]
posted_inflow = sum(row.amount_kopecks for row in posted if row.direction == "in")
posted_outflow = sum(row.amount_kopecks for row in posted if row.direction == "out")
hold_total = sum(abs(row.amount_kopecks) for row in holds)
declared_inflow = getattr(draft, "declared_inflow_kopecks", None)
declared_outflow = getattr(draft, "declared_outflow_kopecks", None)
declared_hold = getattr(draft, "declared_hold_kopecks", None)
full = draft.opening_balance_kopecks is not None and draft.closing_balance_kopecks is not None
checks: list[bool] = []
balance_matched: bool | None = None
computed_closing: int | None = None
if full:
computed_closing = int(draft.opening_balance_kopecks) + posted_inflow - posted_outflow
balance_matched = computed_closing == int(draft.closing_balance_kopecks)
checks.append(balance_matched)
posted_totals_matched: bool | None = None
if declared_inflow is not None or declared_outflow is not None:
posted_totals_matched = (declared_inflow is None or declared_inflow == posted_inflow) and (
declared_outflow is None or declared_outflow == posted_outflow
)
checks.append(posted_totals_matched)
hold_totals_matched: bool | None = None
if declared_hold is not None:
hold_totals_matched = declared_hold == hold_total
checks.append(hold_totals_matched)
return {
"currency": draft.currency,
"opening_balance_kopecks": draft.opening_balance_kopecks,
"closing_balance_kopecks": draft.closing_balance_kopecks,
"full": full,
"declared_inflow_kopecks": declared_inflow,
"declared_outflow_kopecks": declared_outflow,
"declared_hold_kopecks": declared_hold,
"parsed_posted_inflow_kopecks": posted_inflow,
"parsed_posted_outflow_kopecks": posted_outflow,
"parsed_hold_total_kopecks": hold_total,
"posted_totals_matched": posted_totals_matched,
"hold_totals_matched": hold_totals_matched,
"balance_matched": balance_matched,
"computed_closing_kopecks": computed_closing,
"matched": all(checks) if checks else None,
"confirmed_without_full_reconciliation": False,
}
def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun) -> bool:
pending = False
for source in db.query(SourceRecord).filter_by(parse_run_id=run.id).all():
if source.bank_status == "hold":
continue
duplicate = None
if source.external_id:
matches = (
@@ -170,6 +236,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
SourceRecord.space_id == document.space_id,
SourceRecord.id != source.id,
SourceRecord.external_id == source.external_id,
SourceRecord.bank_status == "posted",
ParseRun.adapter_id == run.adapter_id,
)
.all()
@@ -183,6 +250,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
SourceRecord.id != source.id,
SourceRecord.record_fp == source.record_fp,
SourceRecord.direction == source.direction,
SourceRecord.bank_status == "posted",
)
.all()
)
@@ -212,6 +280,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
SourceRecord.id != source.id,
SourceRecord.amount_kopecks == source.amount_kopecks,
SourceRecord.direction == opposite,
SourceRecord.bank_status == "posted",
SourceRecord.occurred_on >= source.occurred_on - timedelta(days=1),
SourceRecord.occurred_on <= source.occurred_on + timedelta(days=1),
)
@@ -234,6 +303,105 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
return pending
def _match_hold_resolutions(
db: Session, *, document: StatementDocument, run: ParseRun
) -> dict[uuid.UUID, uuid.UUID]:
"""Conservatively match unresolved confirmed-doc HOLDs to new posted records.
Candidate pairs are searched only within the same space, mapped account,
and statement contract (adapter). A stable external ID wins; otherwise a
single unambiguous date/amount/direction/normalized-description
fingerprint match is required. Any overlap ambiguity leaves every
involved HOLD unresolved (review-required). Resolutions are written only
when the new statement is confirmed.
"""
if run.adapter_id is None or document.account_id is None:
return {}
holds = (
db.query(SourceRecord)
.join(
StatementDocument,
(StatementDocument.space_id == SourceRecord.space_id)
& (StatementDocument.id == SourceRecord.document_id),
)
.outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id)
.filter(
SourceRecord.space_id == document.space_id,
SourceRecord.bank_status == "hold",
SourceRecord.parse_run_id != run.id,
StatementDocument.status == "confirmed",
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
StatementDocument.adapter_id == run.adapter_id,
StatementDocument.account_id == document.account_id,
HoldResolution.id.is_(None),
)
.all()
)
if not holds:
return {}
posted = (
db.query(SourceRecord)
.filter_by(parse_run_id=run.id, bank_status="posted")
.filter(SourceRecord.review_decision != "discard")
.all()
)
by_external: dict[str, list[SourceRecord]] = {}
by_fingerprint: dict[tuple[str, str], list[SourceRecord]] = {}
for record in posted:
if record.external_id:
by_external.setdefault(record.external_id, []).append(record)
by_fingerprint.setdefault((record.record_fp, record.direction), []).append(record)
candidates: dict[uuid.UUID, set[uuid.UUID]] = {}
for hold in holds:
matched: set[uuid.UUID] = set()
if hold.external_id:
matched.update(item.id for item in by_external.get(hold.external_id, []))
matched.update(item.id for item in by_fingerprint.get((hold.record_fp, hold.direction), []))
if len(matched) == 1:
candidates[hold.id] = matched
claimed: dict[uuid.UUID, int] = {}
for posted_ids in candidates.values():
for posted_id in posted_ids:
claimed[posted_id] = claimed.get(posted_id, 0) + 1
return {
hold_id: next(iter(posted_ids))
for hold_id, posted_ids in candidates.items()
if all(claimed[posted_id] == 1 for posted_id in posted_ids)
}
def _apply_hold_matches(
db: Session,
*,
document: StatementDocument,
run: ParseRun,
user: User,
matches: dict[uuid.UUID, uuid.UUID],
) -> None:
for hold_id, posted_id in matches.items():
db.add(
HoldResolution(
space_id=document.space_id,
hold_source_record_id=hold_id,
resolution="matched_posted",
resolving_source_record_id=posted_id,
resolved_at=utcnow(),
)
)
if matches:
db.flush()
audit.record(
db,
event_type="import.holds_matched_posted",
space_id=document.space_id,
user_id=user.id,
detail={
"document_id": str(document.id),
"holds": sorted(str(hold_id) for hold_id in matches),
},
)
def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> str:
with session_scope(space_id) as db:
document = _document(db, space_id=space_id, document_id=document_id)
@@ -273,23 +441,8 @@ def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> s
document.adapter_id = adapter.adapter_id
document.account_fingerprint = draft.account_fingerprint
document.active_parse_run_id = run.id
document.reconciliation = {
"currency": draft.currency,
"opening_balance_kopecks": draft.opening_balance_kopecks,
"closing_balance_kopecks": draft.closing_balance_kopecks,
"full": draft.opening_balance_kopecks is not None
and draft.closing_balance_kopecks is not None,
"confirmed_without_full_reconciliation": False,
}
if document.reconciliation["full"]:
signed_total = sum(
row.amount_kopecks if row.direction == "in" else -row.amount_kopecks
for row in draft.rows
)
computed = int(draft.opening_balance_kopecks or 0) + signed_total
document.reconciliation["computed_closing_kopecks"] = computed
document.reconciliation["matched"] = computed == draft.closing_balance_kopecks
if draft.account_fingerprint and document.account_id is None:
document.reconciliation = _reconciliation_summary(draft)
if document.account_id is None and draft.account_fingerprint:
matches = (
db.query(Account)
.filter_by(space_id=space_id, fingerprint=draft.account_fingerprint)
@@ -351,6 +504,25 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
.order_by(SourceRecord.created_at)
.all()
)
hold_ids = [record.id for record in records]
resolutions: dict[uuid.UUID, dict[str, Any]] = {}
if hold_ids:
for item in (
db.query(HoldResolution)
.filter(
HoldResolution.space_id == space_id,
HoldResolution.hold_source_record_id.in_(hold_ids),
)
.all()
):
resolutions[item.hold_source_record_id] = {
"resolution": item.resolution,
"resolving_source_record_id": item.resolving_source_record_id,
"operation_id": item.operation_id,
"resolved_by_user_id": item.resolved_by_user_id,
"reason": item.reason,
"resolved_at": item.resolved_at,
}
return {
"id": document.id,
"status": document.status,
@@ -367,11 +539,13 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
"posted_on": item.posted_on,
"amount_kopecks": item.amount_kopecks,
"direction": item.direction,
"bank_status": item.bank_status,
"description": item.description,
"critical": item.critical,
"warnings": item.field_warnings,
"review_decision": item.review_decision,
"duplicate_operation_id": item.duplicate_operation_id,
"hold_resolution": resolutions.get(item.id),
}
for item in records
],
@@ -489,6 +663,119 @@ def set_review_confirmation(
return document
def resolve_hold(
db: Session,
*,
space_id: uuid.UUID,
document_id: uuid.UUID,
source_record_id: uuid.UUID,
user: User,
action: str,
reason: str | None,
account_id: uuid.UUID | None = None,
) -> HoldResolution:
"""Manually settle one HOLD: book it as a normal operation or discard it."""
document = _document(db, space_id=space_id, document_id=document_id)
if document.status != "confirmed":
raise conflict(
"IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED",
"Holds can only be resolved after the import is confirmed",
)
if action not in HOLD_MANUAL_ACTIONS:
raise bad_request("IMPORT_HOLD_ACTION_INVALID", "Unknown hold resolution action")
if reason is None or not reason.strip():
raise conflict("IMPORT_HOLD_REASON_REQUIRED", "A resolution reason is required")
source = (
db.query(SourceRecord)
.filter_by(
id=source_record_id,
space_id=space_id,
parse_run_id=document.active_parse_run_id,
)
.one_or_none()
)
if source is None:
raise not_found("IMPORT_HOLD_NOT_FOUND", "Import row not found")
if source.bank_status != "hold":
raise conflict("IMPORT_HOLD_INVALID", "Only hold rows can be resolved manually")
existing = db.query(HoldResolution.id).filter_by(hold_source_record_id=source.id).one_or_none()
if existing is not None:
raise conflict("IMPORT_HOLD_ALREADY_RESOLVED", "Hold is already resolved")
if source.occurred_on is None:
raise conflict("IMPORT_HOLD_DATE_MISSING", "Hold has no usable operation date")
if source.direction not in {"in", "out"}:
raise conflict("IMPORT_HOLD_DIRECTION_INVALID", "Hold direction is invalid")
resolution = HoldResolution(
space_id=space_id,
hold_source_record_id=source.id,
resolution="confirmed_as_posted"
if action == "confirm_as_posted"
else "discarded_as_cancelled",
resolved_by_user_id=user.id,
reason=reason.strip(),
resolved_at=utcnow(),
)
if action == "confirm_as_posted":
mapped_account_id = account_id or document.account_id
if mapped_account_id is None:
raise conflict(
"IMPORT_HOLD_ACCOUNT_REQUIRED",
"A mapped account is required to confirm a hold as posted",
)
account = db.query(Account).filter_by(id=mapped_account_id, space_id=space_id).one_or_none()
if account is None:
raise not_found("LEDGER_ACCOUNT_NOT_FOUND", "Account not found")
if source.direction == "in":
operation = create_income(
db,
space_id=space_id,
user=user,
account_id=account.id,
amount_kopecks=source.amount_kopecks,
occurred_on=source.occurred_on,
posted_on=source.posted_on,
description=source.description,
request_key=f"import:{source.id}",
)
else:
operation = create_expense(
db,
space_id=space_id,
user=user,
account_id=account.id,
amount_kopecks=source.amount_kopecks,
occurred_on=source.occurred_on,
posted_on=source.posted_on,
description=source.description,
request_key=f"import:{source.id}",
)
db.add(
SourceRecordLink(
space_id=space_id,
source_record_id=source.id,
operation_id=operation.id,
kind="hold_confirmed",
)
)
resolution.operation_id = operation.id
db.add(resolution)
db.flush()
audit.record(
db,
event_type="import.hold_confirmed_as_posted"
if action == "confirm_as_posted"
else "import.hold_discarded_as_cancelled",
space_id=space_id,
user_id=user.id,
detail={
"document_id": str(document.id),
"source_record_id": str(source.id),
"reason": reason.strip(),
},
)
return resolution
def confirm_document(
db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID, user: User
) -> StatementDocument:
@@ -503,7 +790,7 @@ def confirm_document(
"confirmed_without_full_reconciliation"
):
raise conflict("IMPORT_RECONCILIATION_REQUIRED", "Reconciliation must be confirmed")
if reconciliation.get("full") and reconciliation.get("matched") is False:
if reconciliation.get("matched") is False:
raise conflict("IMPORT_RECONCILIATION_MISMATCH", "Statement arithmetic does not reconcile")
pending = (
db.query(DedupCandidate.id)
@@ -519,7 +806,18 @@ def confirm_document(
records = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).all()
if any(item.critical for item in records):
raise conflict("IMPORT_FIELDS_UNRESOLVED", "Critical source fields remain unresolved")
run = db.get(ParseRun, document.active_parse_run_id)
if run is not None:
_apply_hold_matches(
db,
document=document,
run=run,
user=user,
matches=_match_hold_resolutions(db, document=document, run=run),
)
for source in records:
if source.bank_status == "hold":
continue
if source.review_decision == "discard":
continue
if source.review_decision == "transfer":
@@ -755,10 +1053,21 @@ def revert_document(
document = _document(db, space_id=space_id, document_id=document_id)
if document.status != "confirmed":
return document
if document.active_parse_run_id:
db.query(HoldResolution).filter(
HoldResolution.space_id == space_id,
HoldResolution.resolution == "matched_posted",
HoldResolution.resolving_source_record_id.in_(
db.query(SourceRecord.id).filter_by(parse_run_id=document.active_parse_run_id)
),
).delete(synchronize_session=False)
links = (
db.query(SourceRecordLink)
.join(SourceRecord, SourceRecord.id == SourceRecordLink.source_record_id)
.filter(SourceRecord.parse_run_id == document.active_parse_run_id)
.filter(
SourceRecord.parse_run_id == document.active_parse_run_id,
SourceRecordLink.kind != "hold_confirmed",
)
.all()
)
transfer_candidates = (
+4
View File
@@ -39,6 +39,10 @@ package = false
pythonpath = ["."]
testpaths = ["tests"]
filterwarnings = ["ignore::DeprecationWarning"]
addopts = "-m 'not private_fixtures'"
markers = [
"private_fixtures: real-statement private suite; run explicitly with -m private_fixtures (never in CI)",
]
[tool.ruff]
line-length = 100
+825
View File
@@ -0,0 +1,825 @@
"""Sanitized synthetic contract tests for the evidenced statement adapters.
These tests never touch real statements: every input below is synthetic and
tracked. Real-file validation lives in the separately collected private
suite (tests/test_private_fixtures.py, marker `private_fixtures`).
"""
import io
from datetime import date
import pytest
from openpyxl import Workbook
from app.errors import AppError
from app.imports.adapters import ADAPTERS, detect_adapter, magic_kind
from app.imports.adapters import alfa_xlsx as alfa_module
from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter
from app.imports.adapters.base import (
ALFA_DEBIT_XLSX_VERSION,
BANK_STATUS_HOLD,
OZON_DEBIT_PDF_VERSION,
PARSER_VERSION,
ROCKETBANK_DEBIT_PDF_VERSION,
SourceRow,
StatementDraft,
UnsupportedStatementLayout,
account_fingerprint,
money_to_kopecks,
)
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter
from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter
from app.imports.service import _reconciliation_summary
NBSP = "\u00a0"
def make_source_row(**overrides: object) -> SourceRow:
values: dict[str, object] = {
"external_id": None,
"occurred_on": date(2026, 1, 1),
"posted_on": date(2026, 1, 2),
"amount_kopecks": 100,
"direction": "out",
"description": None,
"mcc": None,
"bank_category": None,
"location": None,
"row_ref": "row1",
}
values.update(overrides)
return SourceRow(**values) # type: ignore[arg-type]
def make_draft(**overrides: object) -> StatementDraft:
values: dict[str, object] = {
"adapter_id": "alfa_debit_xlsx",
"parser_version": ALFA_DEBIT_XLSX_VERSION,
"bank_code": "alfa",
"product": "debit",
"account_fingerprint": None,
"currency": "RUB",
"period_start": None,
"period_end": None,
"opening_balance_kopecks": None,
"closing_balance_kopecks": None,
"rows": (),
}
values.update(overrides)
return StatementDraft(**values) # type: ignore[arg-type]
class TestBaseContract:
def test_source_row_defaults_to_posted(self) -> None:
assert make_source_row().bank_status == "posted"
def test_negative_amount_is_rejected(self) -> None:
with pytest.raises(UnsupportedStatementLayout):
make_source_row(amount_kopecks=-1)
def test_unknown_bank_status_is_rejected(self) -> None:
with pytest.raises(UnsupportedStatementLayout):
make_source_row(bank_status="pending")
def test_draft_declared_totals_default_to_none(self) -> None:
draft = make_draft()
assert draft.declared_inflow_kopecks is None
assert draft.declared_outflow_kopecks is None
assert draft.declared_hold_kopecks is None
def test_per_contract_parser_versions(self) -> None:
assert PARSER_VERSION == "1"
assert ALFA_DEBIT_XLSX_VERSION == "2"
assert OZON_DEBIT_PDF_VERSION == "2"
assert ROCKETBANK_DEBIT_PDF_VERSION == "2"
assert AlfaDebitXlsxAdapter.parser_version == ALFA_DEBIT_XLSX_VERSION
assert OzonDebitPdfAdapter.parser_version == OZON_DEBIT_PDF_VERSION
assert RocketbankDebitPdfAdapter.parser_version == ROCKETBANK_DEBIT_PDF_VERSION
def test_registry_lists_only_verified_contracts(self) -> None:
assert [adapter.adapter_id for adapter in ADAPTERS] == [
"alfa_debit_xlsx",
"ozon_debit_pdf",
"rocketbank_debit_pdf",
]
def test_credit_adapters_stay_present_but_unregistered(self) -> None:
assert AlfaCreditXlsxAdapter.adapter_id == "alfa_credit_xlsx"
assert OzonCreditPdfAdapter.adapter_id == "ozon_credit_pdf"
assert all(
adapter.adapter_id not in {"alfa_credit_xlsx", "ozon_credit_pdf"}
for adapter in ADAPTERS
)
def test_detect_adapter_fails_closed_on_unknown_xlsx(self) -> None:
buffer = io.BytesIO()
Workbook().save(buffer)
raw = buffer.getvalue()
with pytest.raises(UnsupportedStatementLayout) as exc:
detect_adapter(raw, "xlsx")
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
def test_detect_adapter_fails_closed_on_unknown_pdf(self) -> None:
with pytest.raises(UnsupportedStatementLayout):
detect_adapter(minimal_pdf(), "pdf")
def test_magic_kind(self) -> None:
assert magic_kind(b"PK\x03\x04") == "xlsx"
assert magic_kind(b"%PDF-1.4") == "pdf"
assert magic_kind(b"not a statement") is None
class TestMoneyParsing:
def test_plain_and_separated_amounts(self) -> None:
assert money_to_kopecks("123") == 12300
assert money_to_kopecks(f"1{NBSP}234,56") == 123456
assert money_to_kopecks("1 234.56") == 123456
assert money_to_kopecks("-1 234,56 RUR") == -123456
assert money_to_kopecks("+ 500.00 \u20bd") == 50000
def test_malformed_amount_fails_closed_without_leaking_digits(self) -> None:
with pytest.raises(UnsupportedStatementLayout) as exc:
money_to_kopecks("12a3,45")
assert "#" in exc.value.message
assert "12" not in exc.value.message
def alfa_workbook_bytes(rows: list[list[object]]) -> bytes:
buffer = io.BytesIO()
workbook = Workbook()
sheet = workbook.worksheets[0]
for row in rows:
sheet.append(row)
workbook.save(buffer)
return buffer.getvalue()
def synthetic_alfa_rows() -> list[list[object]]:
return [
[None, None, None],
["Выписка по счету"],
["За период с 01.09.2026 по 04.09.2026"],
["Номер счета", None, "00000000000000000001"],
["Дата открытия счета", None, "01.01.2025"],
["Валюта счета", None, "RUR"],
["Тип счета", None, "Текущий счёт"],
["Дата формирования выписки", None, "10.09.2026"],
[
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
"Поступления",
None,
"1 000,00 RUR",
],
[None, None, None, None, None, None, None, None, None, None, "Расходы", None, "293,45 RUR"],
[
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
"Неподтвержденные операции",
None,
"100,00 RUR",
],
[
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
"Текущий баланс",
None,
"800,00 RUR",
],
["Операции по счету"],
[
"Дата операции",
"Дата проводки",
None,
"Код",
"Категория",
None,
None,
None,
None,
None,
None,
"Описание",
f"Сумма в{NBSP}валюте{NBSP}счета",
None,
"Статус",
],
[
"01.09.2026",
"01.09.2026",
None,
"000000000000001",
"Покупки",
None,
None,
None,
None,
None,
None,
"Магазин 1 MCC: 5411",
"-123,45",
"Выполнен",
],
[
"02.09.2026",
"02.09.2026",
None,
"000000000000002",
"Переводы",
None,
None,
None,
None,
None,
None,
"Перевод 2 MCC5912",
"1 000,00",
"Выполнен",
],
[
"03.09.2026",
"HOLD",
None,
"000000000000003",
"Покупки",
None,
None,
None,
None,
None,
None,
"Покупка 3",
"-100,00",
],
[
"04.09.2026",
"04.09.2026",
None,
"000000000000004",
"Покупки",
None,
None,
None,
None,
None,
None,
"Покупка 4 MCC 5941",
"-50,00",
"Выполнен",
],
[
"05.09.2026",
"05.09.2026",
None,
"000000000000005",
"Покупки",
None,
None,
None,
None,
None,
None,
"Покупка 5",
"-20,00",
"Выполнен",
],
[None],
[
"(подпись сотрудника АО «БАНК»)",
None,
None,
None,
None,
None,
None,
"Уполномоченное лицо",
],
["Страница 1 из 1"],
]
class TestAlfaDebitContract:
def setup_method(self) -> None:
self.adapter = AlfaDebitXlsxAdapter()
self.raw = alfa_workbook_bytes(synthetic_alfa_rows())
def test_detect(self) -> None:
assert self.adapter.detect(self.raw) is True
def test_detect_rejects_workbook_without_contract_header(self) -> None:
raw = alfa_workbook_bytes([["Выписка по счету"], ["Что-то другое", "123"]])
assert self.adapter.detect(raw) is False
def test_parse_contract_fields(self) -> None:
draft = self.adapter.parse(self.raw)
assert draft.adapter_id == "alfa_debit_xlsx"
assert draft.parser_version == ALFA_DEBIT_XLSX_VERSION
assert draft.bank_code == "alfa"
assert draft.product == "debit"
assert draft.currency == "RUB"
assert draft.account_fingerprint == account_fingerprint("alfa", "00000000000000000001")
assert draft.declared_inflow_kopecks == 100000
# canonical posted-only: raw 293,45 minus declared hold 100,00
assert draft.declared_outflow_kopecks == 19345
assert draft.declared_hold_kopecks == 10000
assert draft.closing_balance_kopecks == 80000
assert draft.opening_balance_kopecks is None
# exact header period fields, not min/max of transaction dates
assert draft.period_start == date(2026, 9, 1)
assert draft.period_end == date(2026, 9, 4)
def test_missing_period_field_fails_closed(self) -> None:
rows = [list(row) for row in synthetic_alfa_rows()]
rows[2] = ["Что-то другое"]
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(alfa_workbook_bytes(rows))
assert "period" in exc.value.message
def test_inconsistent_expense_control_fails_closed(self) -> None:
rows = [list(row) for row in synthetic_alfa_rows()]
rows[9] = [
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
"Расходы",
None,
"50,00 RUR",
]
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(alfa_workbook_bytes(rows))
assert "expense" in exc.value.message
def test_parse_rows_with_hold(self) -> None:
draft = self.adapter.parse(self.raw)
assert len(draft.rows) == 5
first, second, hold, fourth, last = draft.rows
assert (first.amount_kopecks, first.direction, first.bank_status) == (
12345,
"out",
"posted",
)
assert first.posted_on == date(2026, 9, 1)
assert (second.amount_kopecks, second.direction) == (100000, "in")
assert hold.bank_status == BANK_STATUS_HOLD
assert hold.posted_on is None
assert hold.amount_kopecks == 10000
assert hold.direction == "out"
assert last.warnings == ()
assert first.row_ref == "sheet0:row15"
assert first.bank_category == "Покупки"
def test_external_ids_from_code_column(self) -> None:
draft = self.adapter.parse(self.raw)
assert [row.external_id for row in draft.rows] == [
"000000000000001",
"000000000000002",
"000000000000003",
"000000000000004",
"000000000000005",
]
def test_mcc_from_description(self) -> None:
draft = self.adapter.parse(self.raw)
first, second, hold, fourth, last = draft.rows
assert first.mcc == 5411
assert second.mcc == 5912
assert hold.mcc is None
assert fourth.mcc == 5941
assert last.mcc is None
assert all(row.location is None for row in draft.rows)
def test_footer_is_excluded(self) -> None:
draft = self.adapter.parse(self.raw)
assert all(row.occurred_on is not None for row in draft.rows)
assert draft.rows[-1].occurred_on == date(2026, 9, 5)
def test_parse_is_deterministic(self) -> None:
assert self.adapter.parse(self.raw) == self.adapter.parse(self.raw)
def test_service_reconciliation_contract(self) -> None:
draft = self.adapter.parse(self.raw)
summary = _reconciliation_summary(draft)
assert summary["matched"] is True
assert summary["posted_totals_matched"] is True
assert summary["hold_totals_matched"] is True
def test_service_reconciliation_flags_mismatch(self) -> None:
draft = make_draft(
declared_inflow_kopecks=1000,
declared_outflow_kopecks=1,
declared_hold_kopecks=0,
rows=(make_source_row(amount_kopecks=1000, direction="in"),),
)
summary = _reconciliation_summary(draft)
assert summary["posted_totals_matched"] is False
assert summary["matched"] is False
def test_malformed_layout_fails_closed(self) -> None:
raw = alfa_workbook_bytes([["Дата", "Сумма"], ["01.09.2026", "-10,00"]])
assert self.adapter.detect(raw) is False
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(raw)
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
def test_unexpected_mid_table_row_fails_closed(self) -> None:
rows = [list(row) for row in synthetic_alfa_rows()]
rows.insert(16, ["unexpected layout drift"])
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(alfa_workbook_bytes(rows))
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
def test_unsupported_currency_fails_closed(self) -> None:
rows = [list(row) for row in synthetic_alfa_rows()]
rows[5][2] = "USD"
with pytest.raises(UnsupportedStatementLayout):
self.adapter.parse(alfa_workbook_bytes(rows))
def test_row_quota_is_enforced(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(alfa_module, "get_settings_row_cap", lambda: 3)
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(self.raw)
assert "quota" in exc.value.message
def ozon_extracted() -> tuple[list[str], list[list[list[object]]]]:
pages_text = [
"\n".join(
[
"ООО «ОЗОН Банк», ИНН 1234567890, город Москва",
"Справка о движении средств",
"Номер лицевого счёта: № 00000000000000000001, открыт 01.01.2025",
"Период выписки: 01.02.2026 – 28.02.2026",
"Входящий остаток: 1000.50 \u20bd",
]
),
"",
"\n".join(
[
"Итого зачислений за период: 3000.00 \u20bd",
"Итого списаний за период: 2500.75 \u20bd",
"Исходящий остаток: 1499.75 \u20bd",
]
),
]
pages_tables = [
[
[
[
"Дата операции",
"Документ",
"Назначение платежа",
"Сумма операции",
None,
],
[None, None, None, "Российские рубли", "Валюта"],
[
"01.02.2026 10:00:00",
"1234567890\n1",
"Оплата товаров\nпо карте 1234",
"- 500.00 \u20bd",
"- 500.00 \u20bd",
],
]
],
[
[
[
"02.02.2026 11:30:00",
"1234567891",
"Перевод\nчерез СБП",
"+ 2 000,00 \u20bd",
"+ 2 000,00 \u20bd",
]
]
],
[],
]
return pages_text, pages_tables
class TestOzonDebitContract:
def setup_method(self) -> None:
self.adapter = OzonDebitPdfAdapter()
self.pages_text, self.pages_tables = ozon_extracted()
def test_core_contract_fields(self) -> None:
draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables)
assert draft.adapter_id == "ozon_debit_pdf"
assert draft.bank_code == "ozon"
assert draft.currency == "RUB"
assert draft.account_fingerprint == account_fingerprint("ozon", "00000000000000000001")
assert draft.period_start == date(2026, 2, 1)
assert draft.period_end == date(2026, 2, 28)
assert draft.opening_balance_kopecks == 100050
assert draft.closing_balance_kopecks == 149975
assert draft.declared_inflow_kopecks == 300000
assert draft.declared_outflow_kopecks == 250075
assert draft.declared_hold_kopecks is None
def test_multiline_cells_stay_one_transaction(self) -> None:
draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables)
assert len(draft.rows) == 2
first, second = draft.rows
assert first.external_id == "12345678901"
assert first.description == "Оплата товаров по карте 1234"
assert (first.amount_kopecks, first.direction) == (50000, "out")
assert first.occurred_on == date(2026, 2, 1)
assert (second.amount_kopecks, second.direction) == (200000, "in")
assert second.external_id == "1234567891"
assert first.warnings == ()
def test_missing_header_marker_fails_closed(self) -> None:
pages_text, pages_tables = ozon_extracted()
pages_text[0] = "Какой-то другой документ"
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_extracted(pages_text, pages_tables)
def test_missing_totals_fail_closed(self) -> None:
pages_text, pages_tables = ozon_extracted()
pages_text[2] = ""
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_extracted(pages_text, pages_tables)
def test_no_rows_fails_closed(self) -> None:
pages_text, _ = ozon_extracted()
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_extracted(pages_text, [[]])
def test_unexpected_row_shape_fails_closed(self) -> None:
pages_text, pages_tables = ozon_extracted()
pages_tables[0][0].append([None, None, "текст без суммы"])
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_extracted(pages_text, pages_tables)
def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None:
raw = minimal_pdf()
assert self.adapter.detect(raw) is False
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(raw)
assert "text layer" in exc.value.message
def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None:
import app.imports.readers as readers_module
monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings())
with pytest.raises(AppError) as exc:
self.adapter.parse(minimal_pdf(page_count=2))
assert exc.value.code == "IMPORT_QUOTA_EXCEEDED"
def same_page_rocket_lines() -> list[tuple[int, str]]:
return [
(1, "ВЫПИСКА ПО СЧЕТУ"),
(1, "С 01.07.2026 ПО 10.09.2026"),
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
(1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ"),
(1, "0,00 RUR 0,00 RUR"),
(1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"),
(1, "0,00 RUR 0,00 RUR"),
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
(1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
(1, "10:00 00000000001 -10,00 RUR"),
(1, "11:00 00000000002 -20,00 RUR"),
(1, "12:00 00000000003 -30,00 RUR"),
(1, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"),
(1, "ВСЕГО РАСХОДОВ: 60,00 RUR"),
]
def synthetic_rocket_lines() -> list[tuple[int, str]]:
return [
(1, "ВЫПИСКА ПО СЧЕТУ"),
(1, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"),
(1, "С 01.07.2026 ПО 10.09.2026"),
(1, "ИВАНОВ ИВАН ИВАНОВИЧ"),
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
(1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ ДАТА ПОСЛЕДНЕГО ДВИЖЕНИЯ"),
(1, "0,00 RUR 1 000,00 RUR 01.07.2026"),
(1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"),
(1, "500,00 RUR 500,00 RUR"),
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
(1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
(1, "ЗАЧИСЛЕНИЕ ПЕРЕВОДА ДЕНЕЖНЫХ СРЕДСТВ/СБП/"),
(1, "12:00 00000000001 +1 000,00 RUR"),
(1, "СЧЕТ КОРРЕСПОНДЕНТА БАНК ВТБ (ПАО)."),
(1, "ВСЕГО ПОСТУПЛЕНИЙ: 1 000,00 RUR"),
(1, "ВСЕГО РАСХОДОВ: 0,00 RUR"),
(1, "СТРАНИЦА 1/2"),
(2, "ВЫПИСКА ПО СЧЕТУ"),
(2, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"),
(2, "С 01.07.2026 ПО 10.09.2026"),
(2, "02 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 1 000,00 RUR"),
(2, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
(2, "ПЛАТЕЖ АВТОРИЗАЦИЯ №123,01.07.2026 10:00,0000********0000,"),
(2, "13:30 00000000002 555RUR,EKATERINBURG,MCC 5411,SHOP\\RU\\CITY\\SHOP\\ -250,50 RUR"),
(2, "0000\\ ЗДАНИЕ."),
(2, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"),
(2, "ВСЕГО РАСХОДОВ: 250,50 RUR"),
(2, "СТРАНИЦА 2/2"),
]
class TestRocketbankDebitContract:
def setup_method(self) -> None:
self.adapter = RocketbankDebitPdfAdapter()
def test_day_and_page_state_machine(self) -> None:
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
assert len(draft.rows) == 2
first, second = draft.rows
assert first.occurred_on == date(2026, 7, 1)
assert second.occurred_on == date(2026, 7, 2)
def test_row_refs_are_unique_within_a_page(self) -> None:
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
first, second = draft.rows
assert first.row_ref == "page1:record1"
assert second.row_ref == "page2:record1"
same_page = self.adapter._draft_from_lines(same_page_rocket_lines())
refs = [row.row_ref for row in same_page.rows]
assert refs == ["page1:record1", "page1:record2", "page1:record3"]
assert len(set(refs)) == len(refs)
def test_statement_controls(self) -> None:
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
assert draft.adapter_id == "rocketbank_debit_pdf"
assert draft.bank_code == "rocketbank"
assert draft.currency == "RUB"
assert draft.account_fingerprint == account_fingerprint(
"rocketbank", "00000000000000000001"
)
assert draft.period_start == date(2026, 7, 1)
assert draft.period_end == date(2026, 9, 10)
assert draft.opening_balance_kopecks == 0
assert draft.closing_balance_kopecks == 50000
assert draft.declared_inflow_kopecks == 100000
assert draft.declared_outflow_kopecks == 50000
assert draft.declared_hold_kopecks is None
def test_record_folding_and_enrichment(self) -> None:
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
first, second = draft.rows
assert first.external_id == "00000000001"
assert (first.amount_kopecks, first.direction) == (100000, "in")
assert "ЗАЧИСЛЕНИЕ ПЕРЕВОДА" in (first.description or "")
assert "СЧЕТ КОРРЕСПОНДЕНТА" in (first.description or "")
assert first.mcc is None
assert second.external_id == "00000000002"
assert (second.amount_kopecks, second.direction) == (25050, "out")
assert second.mcc == 5411
assert second.location == "EKATERINBURG"
assert (second.description or "").startswith("ПЛАТЕЖ АВТОРИЗАЦИЯ")
assert "555RUR" in (second.description or "")
assert "ЗДАНИЕ." in (second.description or "")
assert all(row.warnings == () for row in draft.rows)
def test_repeated_headers_and_daily_totals_are_ignored(self) -> None:
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
joined = " ".join(row.description or "" for row in draft.rows)
assert "ВСЕГО" not in joined
assert "ВЫПИСКА ПО СЧЕТУ" not in joined
assert "СТРАНИЦА" not in joined
def test_parse_is_deterministic(self) -> None:
assert self.adapter._draft_from_lines(
synthetic_rocket_lines()
) == self.adapter._draft_from_lines(synthetic_rocket_lines())
def test_record_before_day_section_fails_closed(self) -> None:
lines = [
(1, "ВЫПИСКА ПО СЧЕТУ"),
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
(1, "12:00 00000000001 +1 000,00 RUR"),
]
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_lines(lines)
def test_missing_header_block_fails_closed(self) -> None:
lines = [
(1, "ВЫПИСКА ПО СЧЕТУ"),
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
(1, "12:00 00000000001 +1 000,00 RUR"),
]
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_lines(lines)
def test_unrecognized_content_fails_closed(self) -> None:
lines = synthetic_rocket_lines()[:11] + [(1, "НЕИЗВЕСТНАЯ СТРОКА БЕЗ СТРУКТУРЫ")]
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter._draft_from_lines(lines)
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
def test_fold_line_cap_fails_closed(self) -> None:
junk = [(1, f"НЕИЗВЕСТНАЯ СТРОКА {index}") for index in range(7)]
lines = synthetic_rocket_lines()[:11] + junk
with pytest.raises(UnsupportedStatementLayout):
self.adapter._draft_from_lines(lines)
def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None:
raw = minimal_pdf()
assert self.adapter.detect(raw) is False
with pytest.raises(UnsupportedStatementLayout) as exc:
self.adapter.parse(raw)
assert "text layer" in exc.value.message
def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None:
import app.imports.readers as readers_module
monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings())
with pytest.raises(AppError) as exc:
self.adapter.parse(minimal_pdf(page_count=2))
assert exc.value.code == "IMPORT_QUOTA_EXCEEDED"
class _QuotaSettings:
max_pdf_pages = 1
def minimal_pdf(page_count: int = 1, page_text: str = "") -> bytes:
"""Build a small valid single-font PDF with empty or ASCII-only pages."""
objects: list[tuple[int, bytes]] = []
page_ids: list[int] = []
next_id = 3
for _ in range(page_count):
page_id = next_id
content_id = next_id + 1
next_id += 2
page_ids.append(page_id)
content = (
f"BT /F1 12 Tf 72 720 Td ({page_text}) Tj ET".encode("latin-1", "replace")
if page_text
else b""
)
objects.append(
(
content_id,
b"<< /Length %d >>\nstream\n%s\nendstream" % (len(content), content),
)
)
objects.append(
(
page_id,
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents %d 0 R >>"
% content_id,
)
)
font_id = next_id
objects.append((font_id, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"))
objects.insert(
0,
(
2,
b"<< /Type /Pages /Kids ["
+ b" ".join(b"%d 0 R" % page_id for page_id in page_ids)
+ b"] /Count %d >>" % page_count,
),
)
objects.insert(0, (1, b"<< /Type /Catalog /Pages 2 0 R >>"))
out = bytearray(b"%PDF-1.4\n")
offsets: dict[int, int] = {}
for number, body in objects:
offsets[number] = len(out)
out += b"%d 0 obj\n" % number + body + b"\nendobj\n"
xref_position = len(out)
size = max(offsets) + 1
out += b"xref\n0 %d\n" % size
out += b"0000000000 65535 f \n"
for number in range(1, size):
out += b"%010d 00000 n \n" % offsets[number]
out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
size,
xref_position,
)
return bytes(out)
+945
View File
@@ -0,0 +1,945 @@
import uuid
from datetime import date
from types import SimpleNamespace
import pytest
from fastapi.testclient import TestClient
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.budgeting.service import close_period, compute_period_state, get_or_create_period
from app.errors import AppError
from app.identity.models import User
from app.imports import service
from app.imports.adapters.base import record_fingerprint
from app.imports.models import (
HoldResolution,
ParseRun,
SourceRecord,
SourceRecordLink,
StatementDocument,
)
from app.ledger.models import Operation
from app.ledger.service import create_account
from app.spaces.models import Membership
from app.spaces.service import create_space
MAY_10 = date(2026, 5, 10)
def seed(db: Session, *, name: str = "Holds"):
user = User(email=f"{uuid.uuid4().hex}@example.com", password_hash="unused")
db.add(user)
db.flush()
space = create_space(db, user=user, name=name)
account = create_account(
db,
space_id=space.id,
user=user,
name="Debit",
kind="debit",
opened_on=date(2026, 1, 1),
)
return user, space, account
def make_document(
db: Session,
space,
account,
*,
user: User | None = None,
status: str = "ready",
adapter_id: str = "alfa_debit_xlsx",
) -> tuple[StatementDocument, ParseRun]:
document = StatementDocument(
space_id=space.id,
account_id=account.id,
original_filename=f"{uuid.uuid4().hex}.xlsx",
size_bytes=100,
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
object_key=uuid.uuid4().hex,
status=status,
reconciliation={"full": True, "matched": True},
uploaded_by_user_id=user.id if user else None,
)
db.add(document)
db.flush()
run = ParseRun(
space_id=space.id,
document_id=document.id,
adapter_id=adapter_id,
status="succeeded",
)
db.add(run)
db.flush()
document.active_parse_run_id = run.id
document.adapter_id = adapter_id
return document, run
def add_record(
db: Session,
document: StatementDocument,
run: ParseRun,
*,
direction: str = "out",
amount: int = 12_345,
occurred_on: date = MAY_10,
bank_status: str = "posted",
external_id: str | None = None,
description: str | None = None,
) -> SourceRecord:
record = SourceRecord(
space_id=document.space_id,
document_id=document.id,
parse_run_id=run.id,
external_id=external_id,
occurred_on=occurred_on,
amount_kopecks=amount,
direction=direction,
bank_status=bank_status,
description=description,
row_ref=f"sheet0:{uuid.uuid4().hex[:8]}",
record_fp=record_fingerprint(occurred_on, amount, description),
)
db.add(record)
db.flush()
return record
def hold_document(db: Session, *, occurred_on: date = MAY_10, external_id: str | None = None):
"""A confirmed import whose active run contains exactly one unresolved hold."""
user, space, account = seed(db)
document, run = make_document(db, space, account, user=user)
hold = add_record(
db,
document,
run,
bank_status="hold",
occurred_on=occurred_on,
external_id=external_id,
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
db.flush()
return user, space, account, document, hold
def test_bank_status_defaults_to_posted_and_is_constrained(db: Session) -> None:
user, space, account = seed(db)
document, run = make_document(db, space, account, user=user)
record = SourceRecord(
space_id=space.id,
document_id=document.id,
parse_run_id=run.id,
amount_kopecks=1_000,
direction="out",
row_ref="sheet0:row1",
record_fp="fp",
)
db.add(record)
db.flush()
assert record.bank_status == "posted"
bad = SourceRecord(
space_id=space.id,
document_id=document.id,
parse_run_id=run.id,
amount_kopecks=1_000,
direction="out",
bank_status="pending",
row_ref="sheet0:row2",
record_fp="fp2",
)
db.add(bad)
with pytest.raises(IntegrityError):
db.flush()
db.rollback()
def test_hold_row_creates_no_operation_on_confirm(db: Session) -> None:
user, space, account = seed(db)
document, run = make_document(db, space, account, user=user)
hold = add_record(db, document, run, bank_status="hold")
add_record(
db,
document,
run,
bank_status="posted",
occurred_on=date(2026, 5, 11),
amount=7_000,
)
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
db.flush()
operations = db.query(Operation).filter_by(space_id=space.id).all()
assert len(operations) == 1
assert operations[0].amount_kopecks == 7_000
assert db.query(SourceRecordLink).filter_by(source_record_id=hold.id).count() == 0
assert document.status == "confirmed"
detail = service.document_detail(db, space_id=space.id, document_id=document.id)
hold_row = next(item for item in detail["records"] if item["id"] == hold.id)
assert hold_row["bank_status"] == "hold"
assert hold_row["hold_resolution"] is None
assert hold_row["critical"] is False
def test_unresolved_hold_blocks_only_its_occurred_month(db: Session) -> None:
user, space, _account, _document, _hold = hold_document(db)
april = get_or_create_period(db, space_id=space.id, year=2026, month=4)
close_period(db, space_id=space.id, user=user, period=april)
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
state = compute_period_state(db, space_id=space.id, period=may)
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
with pytest.raises(AppError) as exc:
close_period(db, space_id=space.id, user=user, period=may)
assert exc.value.code == "BUDGET_CLOSE_BLOCKED"
june = get_or_create_period(db, space_id=space.id, year=2026, month=6)
june_state = compute_period_state(db, space_id=space.id, period=june)
assert not any(
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in june_state["close_blockers"]
)
def test_discard_as_cancelled_requires_reason_and_unblocks_month(db: Session) -> None:
user, space, _account, document, hold = hold_document(db)
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="discard_as_cancelled",
reason=" ",
)
assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED"
assert db.query(HoldResolution).count() == 0
resolution = service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="discard_as_cancelled",
reason="Cancelled by the bank",
)
db.flush()
assert resolution.resolution == "discarded_as_cancelled"
assert resolution.operation_id is None
assert resolution.resolved_by_user_id == user.id
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
detail = service.document_detail(db, space_id=space.id, document_id=document.id)
hold_row = next(item for item in detail["records"] if item["id"] == hold.id)
assert hold_row["hold_resolution"]["reason"] == "Cancelled by the bank"
april = get_or_create_period(db, space_id=space.id, year=2026, month=4)
close_period(db, space_id=space.id, user=user, period=april)
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
close_period(db, space_id=space.id, user=user, period=may)
assert may.status == "closed"
def test_confirm_as_posted_requires_reason(db: Session) -> None:
user, space, _account, document, hold = hold_document(db)
for reason in (None, ""):
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="confirm_as_posted",
reason=reason,
)
assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED"
assert db.query(HoldResolution).count() == 0
def test_hold_cannot_be_resolved_before_document_confirmation(db: Session) -> None:
user, space, account = seed(db)
document, run = make_document(db, space, account, user=user)
hold = add_record(db, document, run, bank_status="hold")
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="confirm_as_posted",
reason="Not confirmed yet",
)
assert exc.value.code == "IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED"
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
assert db.query(HoldResolution).count() == 0
def test_manual_confirm_rejects_bad_action_and_non_hold_row(db: Session) -> None:
user, space, account = seed(db)
document, run = make_document(db, space, account, user=user)
hold = add_record(db, document, run, bank_status="hold")
posted = add_record(
db, document, run, bank_status="posted", occurred_on=date(2026, 5, 11), amount=900
)
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="delete_forever",
reason="why not",
)
assert exc.value.code == "IMPORT_HOLD_ACTION_INVALID"
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=posted.id,
user=user,
action="discard_as_cancelled",
reason="posted row",
)
assert exc.value.code == "IMPORT_HOLD_INVALID"
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=uuid.uuid4(),
user=user,
action="discard_as_cancelled",
reason="missing row",
)
assert exc.value.code == "IMPORT_HOLD_NOT_FOUND"
def test_manual_confirm_creates_one_operation_and_late_import_dedupes(db: Session) -> None:
user, space, account, document, hold = hold_document(db, external_id="H-1")
resolution = service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="confirm_as_posted",
reason="Verified against the paper receipt",
account_id=account.id,
)
db.flush()
operations = db.query(Operation).filter_by(space_id=space.id).all()
assert len(operations) == 1
operation = operations[0]
assert operation.type == "expense"
assert operation.amount_kopecks == 12_345
assert operation.occurred_on == MAY_10
assert resolution.operation_id == operation.id
link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one_or_none()
assert link is not None and link.operation_id == operation.id
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="discard_as_cancelled",
reason="second try",
)
assert exc.value.code == "IMPORT_HOLD_ALREADY_RESOLVED"
later, later_run = make_document(db, space, account, user=user)
late_posted = add_record(
db,
later,
later_run,
bank_status="posted",
external_id="H-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=later.id, user=user)
db.flush()
assert db.query(Operation).filter_by(space_id=space.id).count() == 1
late_link = db.query(SourceRecordLink).filter_by(source_record_id=late_posted.id).one_or_none()
assert late_link is not None
assert late_link.operation_id == operation.id
assert late_link.kind == "duplicate"
def test_manual_confirm_of_income_hold_creates_income(db: Session) -> None:
user, space, account = seed(db)
document, run = make_document(db, space, account, user=user)
hold = add_record(
db,
document,
run,
direction="in",
amount=45_000,
bank_status="hold",
description="Refund hold",
)
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="confirm_as_posted",
reason="Money arrived",
)
db.flush()
operation = db.query(Operation).filter_by(space_id=space.id).one()
assert operation.type == "income"
assert operation.amount_kopecks == 45_000
def test_manual_confirm_honors_closed_period(db: Session) -> None:
user, space, account = seed(db)
february = get_or_create_period(db, space_id=space.id, year=2026, month=2)
close_period(db, space_id=space.id, user=user, period=february)
march = get_or_create_period(db, space_id=space.id, year=2026, month=3)
close_period(db, space_id=space.id, user=user, period=march)
document, run = make_document(db, space, account, user=user)
hold = add_record(db, document, run, bank_status="hold", occurred_on=date(2026, 3, 15))
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="confirm_as_posted",
reason="Book it",
)
assert exc.value.code == "BUDGET_PERIOD_CLOSED"
db.flush()
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
assert db.query(HoldResolution).count() == 0
resolution = service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="discard_as_cancelled",
reason="Never posted",
)
assert resolution.operation_id is None
def test_unique_posted_record_resolves_hold_on_confirmation(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
hold = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="ALFA-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
assert db.query(HoldResolution).count() == 0
second, second_run = make_document(db, space, account, user=user)
posted = add_record(
db,
second,
second_run,
bank_status="posted",
external_id="ALFA-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
resolution = db.query(HoldResolution).one()
assert resolution.hold_source_record_id == hold.id
assert resolution.resolving_source_record_id == posted.id
assert resolution.resolution == "matched_posted"
assert resolution.resolved_by_user_id is None
operations = db.query(Operation).filter_by(space_id=space.id).all()
assert len(operations) == 1
assert operations[0].amount_kopecks == 12_345
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
state = compute_period_state(db, space_id=space.id, period=may)
assert not any(
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]
)
def test_external_id_match_survives_description_drift(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
hold = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="X-9",
description="HOLD COFFEE",
occurred_on=MAY_10,
)
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
second, second_run = make_document(db, space, account, user=user)
posted = add_record(
db,
second,
second_run,
bank_status="posted",
external_id="X-9",
description="Coffee llc prague",
occurred_on=date(2026, 5, 12),
)
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
resolution = db.query(HoldResolution).one()
assert resolution.hold_source_record_id == hold.id
assert resolution.resolving_source_record_id == posted.id
def test_fingerprint_match_without_external_id_resolves(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
hold = add_record(
db,
first,
first_run,
bank_status="hold",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
second, second_run = make_document(db, space, account, user=user)
posted = add_record(
db,
second,
second_run,
bank_status="posted",
description="Coffee shop",
)
matches = service._match_hold_resolutions(db, document=second, run=second_run)
assert matches == {hold.id: posted.id}
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
assert db.query(HoldResolution).count() == 1
def test_ambiguous_overlap_remains_unresolved(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
hold_a = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="A-1",
description="Coffee shop",
)
hold_b = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="B-2",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
assert hold_a.id != hold_b.id
second, second_run = make_document(db, space, account, user=user)
add_record(
db,
second,
second_run,
bank_status="posted",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
assert db.query(HoldResolution).count() == 0
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
state = compute_period_state(db, space_id=space.id, period=may)
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
def test_matcher_rejects_two_posted_candidates_for_one_hold(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
add_record(db, first, first_run, bank_status="hold", description="Coffee shop")
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
second, second_run = make_document(db, space, account, user=user)
add_record(db, second, second_run, bank_status="posted", description="Coffee shop")
add_record(db, second, second_run, bank_status="posted", description="Coffee shop")
matches = service._match_hold_resolutions(db, document=second, run=second_run)
assert matches == {}
assert db.query(HoldResolution).count() == 0
def test_discarded_posted_record_does_not_resolve_hold(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
hold = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="DISCARDED-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
second, second_run = make_document(db, space, account, user=user)
posted = add_record(
db,
second,
second_run,
bank_status="posted",
external_id="DISCARDED-1",
description="Coffee shop",
)
posted.review_decision = "discard"
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
assert db.query(HoldResolution).filter_by(hold_source_record_id=hold.id).count() == 0
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
state = compute_period_state(db, space_id=space.id, period=may)
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
def test_hold_matching_is_tenant_scoped(db: Session) -> None:
user1, space1, account1 = seed(db, name="Space One")
user2, space2, account2 = seed(db, name="Space Two")
first, first_run = make_document(db, space1, account1, user=user1)
hold = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="T-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space1.id, document_id=first.id, user=user1)
second, second_run = make_document(db, space2, account2, user=user2)
add_record(
db,
second,
second_run,
bank_status="posted",
external_id="T-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space2.id, document_id=second.id, user=user2)
db.flush()
assert db.query(HoldResolution).count() == 0
may_two = get_or_create_period(db, space_id=space2.id, year=2026, month=5)
state_two = compute_period_state(db, space_id=space2.id, period=may_two)
assert not any(
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_two["close_blockers"]
)
may_one = get_or_create_period(db, space_id=space1.id, year=2026, month=5)
state_one = compute_period_state(db, space_id=space1.id, period=may_one)
assert any(
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_one["close_blockers"]
)
with pytest.raises(AppError) as exc:
service.resolve_hold(
db,
space_id=space2.id,
document_id=first.id,
source_record_id=hold.id,
user=user2,
action="discard_as_cancelled",
reason="cross tenant",
)
assert exc.value.code == "IMPORT_NOT_FOUND"
def test_revert_clears_automatic_match_and_reblocks_month(db: Session) -> None:
user, space, account = seed(db)
first, first_run = make_document(db, space, account, user=user)
hold = add_record(
db,
first,
first_run,
bank_status="hold",
external_id="R-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
second, second_run = make_document(db, space, account, user=user)
add_record(
db,
second,
second_run,
bank_status="posted",
external_id="R-1",
description="Coffee shop",
)
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
assert db.query(HoldResolution).count() == 1
service.revert_document(db, space_id=space.id, document_id=second.id, user=user)
db.flush()
assert db.query(HoldResolution).count() == 0
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
state = compute_period_state(db, space_id=space.id, period=may)
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
assert hold.bank_status == "hold"
def test_revert_preserves_manual_hold_operation_and_provenance(db: Session) -> None:
user, space, account, document, hold = hold_document(db)
resolution = service.resolve_hold(
db,
space_id=space.id,
document_id=document.id,
source_record_id=hold.id,
user=user,
action="confirm_as_posted",
reason="Verified in the bank app",
account_id=account.id,
)
db.flush()
operation_id = resolution.operation_id
service.revert_document(db, space_id=space.id, document_id=document.id, user=user)
db.flush()
assert document.status == "ready"
assert operation_id is not None
assert db.get(Operation, operation_id) is not None
assert db.get(HoldResolution, resolution.id) is not None
link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one()
assert link.kind == "hold_confirmed"
assert link.operation_id == operation_id
def test_hold_resolution_rejects_unknown_kind(db: Session) -> None:
user, space, _account, _document, hold = hold_document(db)
db.add(
HoldResolution(
space_id=space.id,
hold_source_record_id=hold.id,
resolution="resolved_somehow",
)
)
with pytest.raises(IntegrityError):
db.flush()
db.rollback()
def test_manual_resolution_requires_reason_at_database_level(db: Session) -> None:
_user, space, _account, _document, hold = hold_document(db)
db.add(
HoldResolution(
space_id=space.id,
hold_source_record_id=hold.id,
resolution="confirmed_as_posted",
reason=None,
)
)
with pytest.raises(IntegrityError):
db.flush()
db.rollback()
_user, space, _account, _document, hold = hold_document(db)
db.add(
HoldResolution(
space_id=space.id,
hold_source_record_id=hold.id,
resolution="matched_posted",
reason=None,
)
)
db.flush()
def _draft(rows, *, opening=1_000, closing=1_200, **declared):
return SimpleNamespace(
currency="RUB",
opening_balance_kopecks=opening,
closing_balance_kopecks=closing,
declared_inflow_kopecks=declared.get("inflow"),
declared_outflow_kopecks=declared.get("outflow"),
declared_hold_kopecks=declared.get("hold"),
rows=tuple(rows),
)
def _row(amount, direction, bank_status="posted"):
return SimpleNamespace(
amount_kopecks=amount,
direction=direction,
bank_status=bank_status,
)
def test_reconciliation_separates_posted_and_hold_totals() -> None:
summary = service._reconciliation_summary(
_draft(
[
_row(50_000, "in"),
_row(30_000, "out"),
_row(20_000, "out", bank_status="hold"),
],
opening=100_000,
closing=120_000,
inflow=50_000,
outflow=30_000,
hold=20_000,
)
)
assert summary["parsed_posted_inflow_kopecks"] == 50_000
assert summary["parsed_posted_outflow_kopecks"] == 30_000
assert summary["parsed_hold_total_kopecks"] == 20_000
assert summary["posted_totals_matched"] is True
assert summary["hold_totals_matched"] is True
assert summary["balance_matched"] is True
assert summary["matched"] is True
assert summary["computed_closing_kopecks"] == 120_000
def test_reconciliation_declared_outflow_mismatch_blocks() -> None:
summary = service._reconciliation_summary(
_draft(
[_row(30_000, "out")],
opening=100_000,
closing=70_000,
inflow=None,
outflow=40_000,
)
)
assert summary["posted_totals_matched"] is False
assert summary["hold_totals_matched"] is None
assert summary["matched"] is False
def test_reconciliation_hold_mismatch_is_review_blocking() -> None:
summary = service._reconciliation_summary(
_draft([_row(25_000, "out", bank_status="hold")], opening=None, closing=None, hold=10_000)
)
assert summary["full"] is False
assert summary["hold_totals_matched"] is False
assert summary["matched"] is False
assert summary["balance_matched"] is None
def test_reconciliation_balance_equation_ignores_holds() -> None:
summary = service._reconciliation_summary(
_draft(
[
_row(50_000, "in"),
_row(30_000, "out"),
_row(20_000, "out", bank_status="hold"),
],
opening=100_000,
closing=140_000,
)
)
assert summary["balance_matched"] is False
assert summary["matched"] is False
assert summary["computed_closing_kopecks"] == 120_000
def test_reconciliation_without_declared_totals_keeps_legacy_semantics() -> None:
draft = SimpleNamespace(
currency="RUB",
opening_balance_kopecks=100,
closing_balance_kopecks=300,
rows=(_row(200, "in"),),
)
summary = service._reconciliation_summary(draft)
assert summary["full"] is True
assert summary["posted_totals_matched"] is None
assert summary["hold_totals_matched"] is None
assert summary["balance_matched"] is True
assert summary["matched"] is True
def test_source_row_without_bank_status_defaults_posted() -> None:
from app.imports.adapters.base import SourceRow
source = service._source_from_row(
space_id=uuid.uuid4(),
document_id=uuid.uuid4(),
run_id=uuid.uuid4(),
row=SourceRow(
external_id=None,
occurred_on=date(2026, 5, 10),
posted_on=None,
amount_kopecks=1_000,
direction="out",
description=None,
mcc=None,
bank_category=None,
location=None,
row_ref="row1",
),
)
assert source.bank_status == "posted"
assert source.critical is False
def test_resolve_endpoint_requires_csrf_and_resolves_hold(client: TestClient, db: Session) -> None:
invite = client.post(
"/api/v1/admin/invitations",
headers={"X-Admin-Token": "test-admin-token"},
json={"email": "holds-user@example.com"},
)
assert invite.status_code == 201, invite.text
accepted = client.post(
"/api/v1/auth/invitations/accept",
json={
"token": invite.json()["token"],
"password": "a-long-password-for-tests",
"fullName": "Holds User",
"locale": "en",
},
)
assert accepted.status_code == 201, accepted.text
csrf = client.cookies["csrf_token"]
db.commit()
user = db.query(User).filter_by(email="holds-user@example.com").one()
space_id = db.query(Membership).filter_by(user_id=user.id).one().space_id
account = create_account(
db,
space_id=space_id,
user=user,
name="Debit",
kind="debit",
opened_on=date(2026, 1, 1),
)
space = SimpleNamespace(id=space_id)
document, run = make_document(db, space, account, user=user)
hold = add_record(db, document, run, bank_status="hold")
service.confirm_document(db, space_id=space_id, document_id=document.id, user=user)
db.commit()
rejected = client.post(
f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve",
json={"action": "confirm_as_posted", "reason": "no csrf"},
)
assert rejected.status_code == 403
resolved = client.post(
f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve",
headers={"X-CSRF-Token": csrf},
json={"action": "confirm_as_posted", "reason": "Checked in the bank app"},
)
assert resolved.status_code == 200, resolved.text
rows = resolved.json()["rows"]
row = next(item for item in rows if item["id"] == str(hold.id))
assert row["bankStatus"] == "hold"
assert row["holdResolution"]["resolution"] == "confirmed_as_posted"
assert row["holdResolution"]["reason"] == "Checked in the bank app"
assert row["holdResolution"]["operationId"] is not None
operations = db.query(Operation).filter_by(space_id=space_id).all()
assert len(operations) == 1
+46 -1
View File
@@ -9,6 +9,7 @@ from sqlalchemy.orm import Session
from app.db.session import set_space_context
from app.imports.blob import get_blob_store
from app.imports.models import HoldResolution, ParseRun, SourceRecord, StatementDocument
from app.ledger.models import Account
from app.spaces.models import FinancialSpace
@@ -28,11 +29,55 @@ def test_postgres_rls_requires_matching_transaction_context() -> None:
db.rollback()
set_space_context(db, space_id)
db.add(Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today()))
account = Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today())
db.add(account)
db.flush()
document = StatementDocument(
space_id=space_id,
account_id=account.id,
original_filename="synthetic.xlsx",
size_bytes=1,
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
object_key=uuid.uuid4().hex,
status="confirmed",
)
db.add(document)
db.flush()
run = ParseRun(
space_id=space_id,
document_id=document.id,
adapter_id="alfa_debit_xlsx",
status="succeeded",
)
db.add(run)
db.flush()
document.active_parse_run_id = run.id
hold = SourceRecord(
space_id=space_id,
document_id=document.id,
parse_run_id=run.id,
occurred_on=date.today(),
amount_kopecks=100,
direction="out",
bank_status="hold",
row_ref="sheet0:row1",
record_fp=uuid.uuid4().hex,
)
db.add(hold)
db.flush()
db.add(
HoldResolution(
space_id=space_id,
hold_source_record_id=hold.id,
resolution="discarded_as_cancelled",
reason="synthetic test resolution",
)
)
db.commit()
set_space_context(db, uuid.uuid4())
assert db.query(Account).count() == 0
assert db.query(HoldResolution).count() == 0
db.rollback()
set_space_context(db, space_id)
persisted_space = db.get(FinancialSpace, space_id)
+224
View File
@@ -0,0 +1,224 @@
"""Private real-statement suite (marker: private_fixtures).
Runs only when invoked explicitly:
PRIVATE_FIXTURES_DIR=/secure/path uv run pytest -m private_fixtures
When PRIVATE_FIXTURES_DIR is unset, the ignored repository-local ``examples/``
directory is used if it contains a ``manifest.json``. Normal CI never collects
this module (the default pytest addopts deselect the marker).
Privacy contract (docs/parsers/private-fixture-gate.md):
- fixture contents, parsed rows, descriptions, identifiers, and manifest
financial values never appear in assertion output, messages, or logs;
- failures report only the filename, adapter id, and the name of the
mismatched aggregate;
- every file is parsed twice and the results must be identical.
"""
import json
import os
from pathlib import Path
import pytest
from app.imports.adapters import detect_adapter, magic_kind
from app.imports.service import _reconciliation_summary
pytestmark = pytest.mark.private_fixtures
REPO_EXAMPLES = Path(__file__).resolve().parents[2] / "examples"
def _load_entries() -> tuple[Path | None, list[dict], str | None]:
"""Resolve fixture entries.
Returns (directory, entries, explicit_error). An explicit error is only
produced when PRIVATE_FIXTURES_DIR was set on purpose: then a missing
directory, missing/malformed manifest, or empty manifest must fail loudly
(redacted message, no values). Without the variable, an absent or broken
repo-local examples directory is a silent skip.
"""
env = os.environ.get("PRIVATE_FIXTURES_DIR")
if env:
directory = Path(env)
if not directory.is_dir():
return None, [], "PRIVATE_FIXTURES_DIR is set but the directory does not exist"
try:
manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
except Exception:
return directory, [], "private manifest.json is missing or unreadable"
entries = manifest.get("fixtures") if isinstance(manifest, dict) else None
if not isinstance(entries, list) or not entries:
return directory, [], "private manifest declares no fixtures"
return directory, [entry for entry in entries if isinstance(entry, dict)], None
try:
if not REPO_EXAMPLES.is_dir():
return None, [], None
manifest = json.loads((REPO_EXAMPLES / "manifest.json").read_text(encoding="utf-8"))
entries = manifest.get("fixtures") if isinstance(manifest, dict) else None
if not isinstance(entries, list):
return None, [], None
return REPO_EXAMPLES, [entry for entry in entries if isinstance(entry, dict)], None
except Exception:
return None, [], None
FIXTURE_DIR, ENTRIES, EXPLICIT_ERROR = _load_entries()
if EXPLICIT_ERROR:
raise pytest.fail.Exception(EXPLICIT_ERROR)
PARAMS = ENTRIES if ENTRIES else [None]
def _check(ok: bool, name: str, what: str) -> None:
assert ok, f"{name}: {what} mismatch"
def _fail(name: str, message: str) -> None:
pytest.fail(f"{name}: {message}", pytrace=False)
@pytest.fixture(params=PARAMS, ids=[entry.get("filename", "unavailable") for entry in PARAMS])
def parsed(request: pytest.FixtureRequest):
entry = request.param
if entry is None:
pytest.skip("private fixtures are not available")
name = str(entry.get("filename", "unnamed"))
path = (FIXTURE_DIR / name) if FIXTURE_DIR is not None else None
if path is None or not path.is_file():
_fail(name, "fixture file is missing")
raw = path.read_bytes()
kind = magic_kind(raw)
if kind not in {"xlsx", "pdf"}:
_fail(name, "unexpected file type")
try:
adapter = detect_adapter(raw, kind)
except Exception:
_fail(name, "adapter detection failed")
if adapter.adapter_id != entry.get("adapter_id"):
_fail(name, "adapter detection mismatch")
try:
first = adapter.parse(raw)
second = adapter.parse(raw)
except Exception:
_fail(name, "parse failed")
if first != second:
_fail(name, "parse result is not deterministic")
return name, entry, first
def test_manifest_contract(parsed) -> None:
name, entry, draft = parsed
_check(draft.parser_version == entry.get("parser_version"), name, "parser version")
for key, actual in (("period_start", draft.period_start), ("period_end", draft.period_end)):
expected = entry.get(key)
_check(
expected is None or (actual is not None and actual.isoformat() == expected), name, key
)
def test_row_counts(parsed) -> None:
name, entry, draft = parsed
rows = draft.rows
_check(len(rows) == entry.get("row_count"), name, "row count")
posted = sum(1 for row in rows if row.bank_status == "posted")
holds = sum(1 for row in rows if row.bank_status == "hold")
_check(posted == entry.get("posted_count"), name, "posted count")
_check(holds == entry.get("hold_count"), name, "hold count")
_check(
sum(1 for row in rows if row.direction == "in") == entry.get("in_count"),
name,
"inflow row count",
)
_check(
sum(1 for row in rows if row.direction == "out") == entry.get("out_count"),
name,
"outflow row count",
)
def test_row_invariants(parsed) -> None:
name, _, draft = parsed
rows = draft.rows
_check(all(row.amount_kopecks >= 0 for row in rows), name, "positive amount invariant")
_check(
all(row.bank_status in {"posted", "hold"} for row in rows),
name,
"bank status invariant",
)
def test_hold_totals(parsed) -> None:
name, entry, draft = parsed
hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold")
_check(hold_total == entry.get("hold_total_kopecks"), name, "hold total")
def test_control_totals(parsed) -> None:
name, entry, draft = parsed
_check(
draft.declared_inflow_kopecks == entry.get("declared_inflow_kopecks"),
name,
"declared inflow total",
)
_check(
draft.declared_outflow_kopecks == entry.get("declared_outflow_kopecks"),
name,
"declared outflow total",
)
_check(
draft.declared_hold_kopecks == entry.get("declared_hold_kopecks"),
name,
"declared hold total",
)
def test_balances(parsed) -> None:
name, entry, draft = parsed
_check(
draft.opening_balance_kopecks == entry.get("opening_balance_kopecks"),
name,
"opening balance",
)
_check(
draft.closing_balance_kopecks == entry.get("closing_balance_kopecks"),
name,
"closing balance",
)
def test_declared_totals_match_parsed_rows(parsed) -> None:
name, _, draft = parsed
posted_in = sum(
row.amount_kopecks
for row in draft.rows
if row.bank_status == "posted" and row.direction == "in"
)
posted_out = sum(
row.amount_kopecks
for row in draft.rows
if row.bank_status == "posted" and row.direction == "out"
)
hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold")
# Declared controls are canonicalized to posted-only by the adapters; the
# hold control equals the parsed hold sum.
_check(posted_in == draft.declared_inflow_kopecks, name, "declared vs parsed inflow")
_check(posted_out == draft.declared_outflow_kopecks, name, "declared vs parsed outflow")
_check(hold_total == (draft.declared_hold_kopecks or 0), name, "declared vs parsed hold")
def test_service_reconciliation_matches(parsed) -> None:
name, _, draft = parsed
summary = _reconciliation_summary(draft)
_check(summary.get("matched") is True, name, "service reconciliation matched")
_check(
summary.get("posted_totals_matched") is True,
name,
"service reconciliation posted totals",
)
_check(
summary.get("hold_totals_matched") is not False,
name,
"service reconciliation hold totals",
)